From 4f0c4a53c7df3c5c6f43190e659bb2bfd8c3827f Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 10 Mar 2023 17:08:29 +0800 Subject: [PATCH 01/88] chore: upgrade dashboard to v1.1.9 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f7ab5244..998175eea 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.8 +export EMQX_DASHBOARD_VERSION ?= v1.1.9 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.4 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From 686bf8255b4a18f524d7a95983e27d094a42ad33 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Mar 2023 14:35:08 +0300 Subject: [PATCH 02/88] fix(bridge): reply `emqx_resource:get_instance/1` from cache The resource manager may be busy at times, so this change ensures that getting resource instance state will not block. Currently, no users of `emqx_resource:get_instance/1` do seem to be relying on state being "as-actual-as-possible" guarantee it was providing. --- apps/emqx_resource/src/emqx_resource.erl | 2 +- .../src/emqx_resource_manager.erl | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 1c5eecfbb..2c6865e04 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -309,7 +309,7 @@ set_resource_status_connecting(ResId) -> -spec get_instance(resource_id()) -> {ok, resource_group(), resource_data()} | {error, Reason :: term()}. get_instance(ResId) -> - emqx_resource_manager:lookup(ResId). + emqx_resource_manager:ets_lookup(ResId, [metrics]). -spec fetch_creation_opts(map()) -> creation_opts(). fetch_creation_opts(Opts) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index ee9e218b2..05d100913 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -36,6 +36,7 @@ list_all/0, list_group/1, ets_lookup/1, + ets_lookup/2, get_metrics/1, reset_metrics/1 ]). @@ -229,14 +230,25 @@ set_resource_status_connecting(ResId) -> -spec lookup(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. lookup(ResId) -> case safe_call(ResId, lookup, ?T_LOOKUP) of - {error, timeout} -> ets_lookup(ResId); + {error, timeout} -> ets_lookup(ResId, [metrics]); Result -> Result end. -%% @doc Lookup the group and data of a resource +%% @doc Lookup the group and data of a resource from the cache -spec ets_lookup(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. ets_lookup(ResId) -> + ets_lookup(ResId, []). + +%% @doc Lookup the group and data of a resource from the cache +-spec ets_lookup(resource_id(), [Option]) -> + {ok, resource_group(), resource_data()} | {error, not_found} +when + Option :: metrics. +ets_lookup(ResId, Options) -> + NeedMetrics = lists:member(metrics, Options), case read_cache(ResId) of + {Group, Data} when NeedMetrics -> + {ok, Group, data_record_to_external_map_with_metrics(Data)}; {Group, Data} -> {ok, Group, data_record_to_external_map(Data)}; not_found -> @@ -253,7 +265,7 @@ reset_metrics(ResId) -> emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId). %% @doc Returns the data for all resources --spec list_all() -> [resource_data()] | []. +-spec list_all() -> [resource_data()]. list_all() -> try [ From 53bc27e0f43db534f9502c2304ac218feea09f59 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Mar 2023 14:49:38 +0300 Subject: [PATCH 03/88] refactor(bridge): avoid unnecessary `maps:to_list/1` when listing --- apps/emqx_bridge/src/emqx_bridge.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index ddf24d380..292369d36 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -226,21 +226,21 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnv) -> Result. list() -> - lists:foldl( - fun({Type, NameAndConf}, Bridges) -> - lists:foldl( - fun({Name, RawConf}, Acc) -> + maps:fold( + fun(Type, NameAndConf, Bridges) -> + maps:fold( + fun(Name, RawConf, Acc) -> case lookup(Type, Name, RawConf) of {error, not_found} -> Acc; {ok, Res} -> [Res | Acc] end end, Bridges, - maps:to_list(NameAndConf) + NameAndConf ) end, [], - maps:to_list(emqx:get_raw_config([bridges], #{})) + emqx:get_raw_config([bridges], #{}) ). lookup(Id) -> From 27d03770fee138f79881a58c4fa96778a88a9cd6 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 10 Mar 2023 17:08:29 +0800 Subject: [PATCH 04/88] chore: upgrade dashboard to v1.1.9 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f7ab5244..998175eea 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.8 +export EMQX_DASHBOARD_VERSION ?= v1.1.9 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.4 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From beef7bb0e7f20314e588dbc7d91536ecb06bff10 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 14 Mar 2023 10:39:56 +0100 Subject: [PATCH 05/88] chore: bump vsn e5.0.2-alpha.1 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index a79389ecb..cdf2eefa7 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.20"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.1"). +-define(EMQX_RELEASE_EE, "5.0.2-alpha.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 8474a00b0..4b5382090 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.1 +version: 5.0.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.1 +appVersion: 5.0.2 From f9556e8e21ebd04d40328bdd00a8c98fd21107c3 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 10 Mar 2023 17:08:29 +0800 Subject: [PATCH 06/88] chore: upgrade dashboard to v1.1.9 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f7ab5244..998175eea 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.8 +export EMQX_DASHBOARD_VERSION ?= v1.1.9 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.4 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From 255117f2c4ff0b12a07d5eee18498641ffa9aacf Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 14 Mar 2023 10:39:56 +0100 Subject: [PATCH 07/88] chore: bump vsn e5.0.2-alpha.1 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index a79389ecb..cdf2eefa7 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.20"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.1"). +-define(EMQX_RELEASE_EE, "5.0.2-alpha.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 8474a00b0..4b5382090 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.1 +version: 5.0.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.1 +appVersion: 5.0.2 From d337814c089c32ee4469d40717b973b5ecb26382 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 14 Mar 2023 16:37:50 +0100 Subject: [PATCH 08/88] ci: tmp fix for build packages - do not build raspbian9 and raspbian10 packages - install krb5-devel in el9 (to be fixed in builder) --- .github/workflows/build_packages.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 73a2ece7a..f98e4a6dc 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -213,8 +213,6 @@ jobs: - ubuntu18.04 - debian11 - debian10 - - raspbian10 - - raspbian9 - el9 - el8 - el7 @@ -231,10 +229,6 @@ jobs: build_machine: ubuntu-22.04 - arch: amd64 build_machine: aws-arm64 - - arch: amd64 - os: raspbian9 - - arch: amd64 - os: raspbian10 include: - profile: emqx otp: 25.1.2-2 @@ -266,6 +260,11 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip + - name: tmp fix for el9 + if: matrix.os == 'el9' + run: | + set -eu + dnf install -y krb5-devel - name: build emqx packages working-directory: source env: @@ -378,8 +377,6 @@ jobs: push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-arm64.deb" push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-amd64.deb" push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-arm64.deb" - push "raspbian/stretch" "packages/$PROFILE/$PROFILE-$VERSION-raspbian9-arm64.deb" - push "raspbian/buster" "packages/$PROFILE/$PROFILE-$VERSION-raspbian10-arm64.deb" push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" From 4880a849b99e4fa74923f937a70d5d13a2e93646 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 3 Mar 2023 14:11:08 -0300 Subject: [PATCH 09/88] chore: update emqtt -> 1.8.5 Needed for OCSP / CRL tests because of a bug that makes emqtt hang forever on TLS handshake errors. --- apps/emqx/rebar.config | 2 +- apps/emqx_retainer/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 949a7a734..e782f714e 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 7e791f90f..65de71fdd 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -27,7 +27,7 @@ {profiles, [ {test, [ {deps, [ - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.5.0"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} ]} ]} ]}. diff --git a/mix.exs b/mix.exs index 12dbd09dc..f819380e2 100644 --- a/mix.exs +++ b/mix.exs @@ -61,7 +61,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.8.2", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.8.5", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config b/rebar.config index c3296518d..c2b500cda 100644 --- a/rebar.config +++ b/rebar.config @@ -63,7 +63,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} From 65fee34fe489cc59ddec13e9294f70ff4af5a778 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 3 Mar 2023 15:15:30 -0300 Subject: [PATCH 10/88] test: fix inter-suite test teardowns --- apps/emqx/src/emqx_config.erl | 9 +++++++++ apps/emqx/test/emqx_SUITE.erl | 1 + apps/emqx/test/emqx_common_test_helpers.erl | 10 ++++++++++ apps/emqx_authz/test/emqx_authz_file_SUITE.erl | 2 +- apps/emqx_authz/test/emqx_authz_http_SUITE.erl | 2 +- apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl | 2 +- apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl | 2 +- apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl | 2 +- .../test/emqx_authz_postgresql_SUITE.erl | 2 +- apps/emqx_authz/test/emqx_authz_redis_SUITE.erl | 2 +- apps/emqx_exhook/test/emqx_exhook_SUITE.erl | 15 ++++++++++++--- .../test/emqx_gateway_authn_SUITE.erl | 2 +- 12 files changed, 40 insertions(+), 11 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index b76234e85..bf3134568 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -87,6 +87,10 @@ remove_handlers/0 ]). +-ifdef(TEST). +-export([erase_schema_mod_and_names/0]). +-endif. + -include("logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -501,6 +505,11 @@ save_schema_mod_and_names(SchemaMod) -> names => lists:usort(OldNames ++ RootNames) }). +-ifdef(TEST). +erase_schema_mod_and_names() -> + persistent_term:erase(?PERSIS_SCHEMA_MODS). +-endif. + -spec get_schema_mod() -> #{binary() => atom()}. get_schema_mod() -> maps:get(mods, persistent_term:get(?PERSIS_SCHEMA_MODS, #{mods => #{}})). diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index dbe8e09a6..09d5d8017 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -26,6 +26,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), Config. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ce3a39dcf..8353c2895 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -16,6 +16,8 @@ -module(emqx_common_test_helpers). +-include("emqx_authentication.hrl"). + -type special_config_handler() :: fun(). -type apps() :: list(atom()). @@ -283,6 +285,14 @@ generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> -spec stop_apps(list()) -> ok. stop_apps(Apps) -> [application:stop(App) || App <- Apps ++ [emqx, ekka, mria, mnesia]], + %% to avoid inter-suite flakiness + application:unset_env(emqx, init_config_load_done), + persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY), + emqx_config:erase_schema_mod_and_names(), + ok = emqx_config:delete_override_conf_files(), + application:unset_env(emqx, local_override_conf_file), + application:unset_env(emqx, cluster_override_conf_file), + application:unset_env(gen_rpc, port_discovery), ok. proj_root() -> diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index 5b5d2618c..124fe904f 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -55,7 +55,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index e91da9829..0757113f3 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -52,7 +52,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), ok = stop_apps([emqx_resource, cowboy]), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index d7b31b5b5..2b7fce309 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -36,7 +36,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl index c685e8237..9ffeacf45 100644 --- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl @@ -50,7 +50,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index ad9a23377..d276a2e1b 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -57,7 +57,7 @@ end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), ok = emqx_resource:remove_local(?MYSQL_RESOURCE), ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index f17c2de1c..0ef21360c 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -57,7 +57,7 @@ end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), ok = emqx_resource:remove_local(?PGSQL_RESOURCE), ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index b480e0262..d68ea342e 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -58,7 +58,7 @@ end_per_suite(_Config) -> ok = emqx_authz_test_lib:restore_authorizers(), ok = emqx_resource:remove_local(?REDIS_RESOURCE), ok = stop_apps([emqx_resource]), - ok = emqx_common_test_helpers:stop_apps([emqx_authz]). + ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index aaa8649c0..d12f99917 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -24,13 +24,13 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx_conf/include/emqx_conf.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(DEFAULT_CLUSTER_NAME_ATOM, emqxcl). -define(OTHER_CLUSTER_NAME_ATOM, test_emqx_cluster). -define(OTHER_CLUSTER_NAME_STRING, "test_emqx_cluster"). --define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard). -define(CONF_DEFAULT, << "\n" @@ -54,6 +54,8 @@ "}\n" >>). +-import(emqx_common_test_helpers, [on_exit/1]). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -89,7 +91,7 @@ init_per_testcase(_, Config) -> timer:sleep(200), Config. -end_per_testcase(_, Config) -> +end_per_testcase(_, _Config) -> case erlang:whereis(node()) of undefined -> ok; @@ -97,7 +99,8 @@ end_per_testcase(_, Config) -> erlang:unlink(P), erlang:exit(P, kill) end, - Config. + emqx_common_test_helpers:call_janitor(), + ok. load_cfg(Cfg) -> ok = emqx_common_test_helpers:load_config(emqx_exhook_schema, Cfg). @@ -300,6 +303,12 @@ t_cluster_name(_) -> emqx_common_test_helpers:stop_apps([emqx, emqx_exhook]), emqx_common_test_helpers:start_apps([emqx, emqx_exhook], SetEnvFun), + on_exit(fun() -> + emqx_common_test_helpers:stop_apps([emqx, emqx_exhook]), + load_cfg(?CONF_DEFAULT), + emqx_common_test_helpers:start_apps([emqx_exhook]), + mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]) + end), ?assertEqual(?OTHER_CLUSTER_NAME_STRING, emqx_sys:cluster_name()), diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 208262f22..2427a10ee 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -77,7 +77,7 @@ init_per_suite(Config) -> end_per_suite(Config) -> emqx_gateway_auth_ct:stop(), emqx_config:erase(gateway), - emqx_mgmt_api_test_util:end_suite([cowboy, emqx_authn, emqx_gateway]), + emqx_mgmt_api_test_util:end_suite([cowboy, emqx_conf, emqx_authn, emqx_gateway]), Config. init_per_testcase(_Case, Config) -> From 422597a441a8f62382ac93c59201410eae3d70a5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 10:32:35 -0300 Subject: [PATCH 11/88] test: fix flaky tests --- .../test/emqx_resource_SUITE.erl | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index af72e86f9..c8b5ff183 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -250,8 +250,15 @@ t_batch_query_counter(_) -> ok end, fun(Trace) -> - QueryTrace = ?of_kind(call_batch_query, Trace), - ?assertMatch([#{batch := BatchReq} | _] when length(BatchReq) > 1, QueryTrace) + QueryTrace = [ + Event + || Event = #{ + ?snk_kind := call_batch_query, + batch := BatchReq + } <- Trace, + length(BatchReq) > 1 + ], + ?assertMatch([_ | _], QueryTrace) end ), {ok, NMsgs} = emqx_resource:query(?ID, get_counter), @@ -602,19 +609,18 @@ t_query_counter_async_inflight_batch(_) -> 5_000 ), fun(Trace) -> - QueryTrace = ?of_kind(call_batch_query_async, Trace), - ?assertMatch( - [ - #{ - batch := [ - {query, _, {inc_counter, 1}, _, _}, - {query, _, {inc_counter, 1}, _, _} - ] - } - | _ - ], - QueryTrace - ) + QueryTrace = [ + Event + || Event = #{ + ?snk_kind := call_batch_query_async, + batch := [ + {query, _, {inc_counter, 1}, _, _}, + {query, _, {inc_counter, 1}, _, _} + ] + } <- + Trace + ], + ?assertMatch([_ | _], QueryTrace) end ), tap_metrics(?LINE), From 52263a044865ba6d7262d2945c83d69432b9af6f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 27 Feb 2023 17:59:37 -0300 Subject: [PATCH 12/88] feat: add ocsp stapling and crl support to mqtt ssl listener --- apps/emqx/i18n/emqx_schema_i18n.conf | 57 ++ apps/emqx/src/emqx_const_v1.erl | 24 + apps/emqx/src/emqx_kernel_sup.erl | 3 +- apps/emqx/src/emqx_listeners.erl | 13 +- apps/emqx/src/emqx_ocsp_cache.erl | 521 ++++++++++ apps/emqx/src/emqx_schema.erl | 122 ++- apps/emqx/test/emqx_common_test_helpers.erl | 37 +- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 908 ++++++++++++++++++ .../test/emqx_ocsp_cache_SUITE_data/ca.pem | 68 ++ .../emqx_ocsp_cache_SUITE_data/client.key | 52 + .../emqx_ocsp_cache_SUITE_data/client.pem | 38 + .../test/emqx_ocsp_cache_SUITE_data/index.txt | 6 + .../ocsp-issuer.key | 52 + .../ocsp-issuer.pem | 34 + .../openssl_listeners.conf | 14 + .../emqx_ocsp_cache_SUITE_data/server.key | 28 + .../emqx_ocsp_cache_SUITE_data/server.pem | 35 + .../test/emqx_mgmt_api_test_util.erl | 9 +- changes/ce/feat-10067.en.md | 1 + changes/ce/feat-10067.zh.md | 1 + scripts/spellcheck/dicts/emqx.txt | 2 + 21 files changed, 1999 insertions(+), 26 deletions(-) create mode 100644 apps/emqx/src/emqx_const_v1.erl create mode 100644 apps/emqx/src/emqx_ocsp_cache.erl create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE.erl create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/ca.pem create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.key create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.pem create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/index.txt create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.key create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.pem create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.key create mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.pem create mode 100644 changes/ce/feat-10067.en.md create mode 100644 changes/ce/feat-10067.zh.md diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index b57698327..0690919bb 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1753,6 +1753,63 @@ server_ssl_opts_schema_gc_after_handshake { } } +server_ssl_opts_schema_enable_ocsp_stapling { + desc { + en: "Whether to enable OCSP stapling for the listener. If set to true," + " requires defining the OCSP responder URL and issuer PEM path." + zh: "是否为监听器启用OCSP装订功能。 如果设置为 true，" + "需要定义OCSP响应者URL和发行者PEM路径。" + } + label: { + en: "Enable OCSP Stapling" + zh: "启用OCSP订书机" + } +} + +server_ssl_opts_schema_ocsp_responder_url { + desc { + en: "URL for the OCSP responder to check the server certificate against." + zh: "用于检查服务器证书的OCSP响应器的URL。" + } + label: { + en: "OCSP Responder URL" + zh: "OCSP响应者URL" + } +} + +server_ssl_opts_schema_ocsp_issuer_pem { + desc { + en: "PEM-encoded certificate of the OCSP issuer for the server certificate." + zh: "服务器证书的OCSP签发者的PEM编码证书。" + } + label: { + en: "OCSP Issuer Certificate" + zh: "OCSP发行人证书" + } +} + +server_ssl_opts_schema_ocsp_refresh_interval { + desc { + en: "The period to refresh the OCSP response for the server." + zh: "为服务器刷新OCSP响应的周期。" + } + label: { + en: "OCSP Refresh Interval" + zh: "OCSP刷新间隔" + } +} + +server_ssl_opts_schema_ocsp_refresh_http_timeout { + desc { + en: "The timeout for the HTTP request when checking OCSP responses." + zh: "检查OCSP响应时，HTTP请求的超时。" + } + label: { + en: "OCSP Refresh HTTP Timeout" + zh: "OCSP刷新HTTP超时" + } +} + fields_listeners_tcp { desc { en: """TCP listeners.""" diff --git a/apps/emqx/src/emqx_const_v1.erl b/apps/emqx/src/emqx_const_v1.erl new file mode 100644 index 000000000..aef4d5101 --- /dev/null +++ b/apps/emqx/src/emqx_const_v1.erl @@ -0,0 +1,24 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @doc Never update this module, create a v2 instead. +%%-------------------------------------------------------------------- + +-module(emqx_const_v1). + +-export([make_sni_fun/1]). + +make_sni_fun(ListenerID) -> + fun(SN) -> emqx_ocsp_cache:sni_fun(SN, ListenerID) end. diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl index a69674de8..9d2f71068 100644 --- a/apps/emqx/src/emqx_kernel_sup.erl +++ b/apps/emqx/src/emqx_kernel_sup.erl @@ -35,7 +35,8 @@ init([]) -> child_spec(emqx_hooks, worker), child_spec(emqx_stats, worker), child_spec(emqx_metrics, worker), - child_spec(emqx_authn_authz_metrics_sup, supervisor) + child_spec(emqx_authn_authz_metrics_sup, supervisor), + child_spec(emqx_ocsp_cache, worker) ] }}. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 6982b3dea..97bc15ad3 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -484,8 +484,12 @@ esockd_opts(ListenerId, Type, Opts0) -> }, maps:to_list( case Type of - tcp -> Opts3#{tcp_options => tcp_opts(Opts0)}; - ssl -> Opts3#{ssl_options => ssl_opts(Opts0), tcp_options => tcp_opts(Opts0)} + tcp -> + Opts3#{tcp_options => tcp_opts(Opts0)}; + ssl -> + OptsWithSNI = inject_sni_fun(ListenerId, Opts0), + SSLOpts = ssl_opts(OptsWithSNI), + Opts3#{ssl_options => SSLOpts, tcp_options => tcp_opts(Opts0)} end ). @@ -785,3 +789,8 @@ quic_listener_optional_settings() -> max_binding_stateless_operations, stateless_operation_expiration_ms ]. + +inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapling := true}}}) -> + emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf); +inject_sni_fun(_ListenerId, Conf) -> + Conf. diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl new file mode 100644 index 000000000..2fe3aa5d5 --- /dev/null +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -0,0 +1,521 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @doc EMQX OCSP cache. +%%-------------------------------------------------------------------- + +-module(emqx_ocsp_cache). + +-include("logger.hrl"). +-include_lib("public_key/include/public_key.hrl"). +-include_lib("ssl/src/ssl_handshake.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-behaviour(gen_server). + +-export([ + start_link/0, + sni_fun/2, + fetch_response/1, + register_listener/2, + inject_sni_fun/2 +]). + +%% gen_server API +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/3 +]). + +%% internal export; only for mocking in tests +-export([http_get/2]). + +-define(CACHE_TAB, ?MODULE). +-define(CALL_TIMEOUT, 20_000). +-define(RETRY_TIMEOUT, 5_000). +-define(REFRESH_TIMER(LID), {refresh_timer, LID}). +-ifdef(TEST). +-define(MIN_REFRESH_INTERVAL, timer:seconds(5)). +-else. +-define(MIN_REFRESH_INTERVAL, timer:minutes(1)). +-endif. + +-define(WITH_LISTENER_CONFIG(ListenerID, ConfPath, Pattern, ErrorResp, Action), + case emqx_listeners:parse_listener_id(ListenerID) of + {ok, #{type := Type, name := Name}} -> + case emqx_config:get_listener_conf(Type, Name, ConfPath, not_found) of + not_found -> + ?SLOG(error, #{ + msg => "listener_config_missing", + listener_id => ListenerID + }), + (ErrorResp); + Pattern -> + Action; + OtherConfig -> + ?SLOG(error, #{ + msg => "listener_config_inconsistent", + listener_id => ListenerID, + config => OtherConfig + }), + (ErrorResp) + end; + _Err -> + ?SLOG(error, #{ + msg => "listener_id_not_found", + listener_id => ListenerID + }), + (ErrorResp) + end +). + +%% Allow usage of OTP certificate record fields (camelCase). +-elvis([ + {elvis_style, atom_naming_convention, #{ + regex => "^([a-z][a-z0-9]*_?)([a-zA-Z0-9]*_?)*$", + enclosed_atoms => ".*" + }} +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +sni_fun(_ServerName, ListenerID) -> + Res = + try + fetch_response(ListenerID) + catch + _:_ -> error + end, + case Res of + {ok, Response} -> + [ + {certificate_status, #certificate_status{ + status_type = ?CERTIFICATE_STATUS_TYPE_OCSP, + response = Response + }} + ]; + error -> + [] + end. + +fetch_response(ListenerID) -> + case do_lookup(ListenerID) of + {ok, DERResponse} -> + {ok, DERResponse}; + {error, invalid_listener_id} -> + error; + {error, not_cached} -> + ?tp(ocsp_cache_miss, #{listener_id => ListenerID}), + ?SLOG(debug, #{ + msg => "fetching_new_ocsp_response", + listener_id => ListenerID + }), + http_fetch(ListenerID) + end. + +register_listener(ListenerID, Opts) -> + gen_server:call(?MODULE, {register_listener, ListenerID, Opts}, ?CALL_TIMEOUT). + +-spec inject_sni_fun(emqx_listeners:listener_id(), map()) -> map(). +inject_sni_fun(ListenerID, Conf0) -> + SNIFun = emqx_const_v1:make_sni_fun(ListenerID), + Conf = emqx_map_lib:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), + ok = ?MODULE:register_listener(ListenerID, Conf), + Conf. + +%%-------------------------------------------------------------------- +%% gen_server behaviour +%%-------------------------------------------------------------------- + +init(_Args) -> + logger:set_process_metadata(#{domain => [emqx, ocsp, cache]}), + _ = ets:new(?CACHE_TAB, [ + named_table, + protected, + {read_concurrency, true} + ]), + ?tp(ocsp_cache_init, #{}), + {ok, #{}}. + +handle_call({http_fetch, ListenerID}, _From, State) -> + case do_lookup(ListenerID) of + {ok, DERResponse} -> + {reply, {ok, DERResponse}, State}; + {error, invalid_listener_id} -> + {reply, error, State}; + {error, not_cached} -> + Conf = undefined, + with_refresh_params(ListenerID, Conf, {reply, error, State}, fun(Params) -> + case do_http_fetch_and_cache(ListenerID, Params) of + error -> {reply, error, ensure_timer(ListenerID, State, ?RETRY_TIMEOUT)}; + {ok, Response} -> {reply, {ok, Response}, ensure_timer(ListenerID, State)} + end + end) + end; +handle_call({register_listener, ListenerID, Conf}, _From, State0) -> + ?SLOG(debug, #{ + msg => "registering_ocsp_cache", + listener_id => ListenerID + }), + RefreshInterval0 = emqx_map_lib:deep_get([ssl_options, ocsp, refresh_interval], Conf), + RefreshInterval = max(RefreshInterval0, ?MIN_REFRESH_INTERVAL), + State = State0#{{refresh_interval, ListenerID} => RefreshInterval}, + %% we need to pass the config along because this might be called + %% during the listener's `post_config_update', hence the config is + %% not yet "commited" and accessible when we need it. + Message = {refresh, ListenerID, Conf}, + {reply, ok, ensure_timer(ListenerID, Message, State, 0)}; +handle_call(Call, _From, State) -> + {reply, {error, {unknown_call, Call}}, State}. + +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info({timeout, TRef, {refresh, ListenerID}}, State0) -> + case maps:get(?REFRESH_TIMER(ListenerID), State0, undefined) of + TRef -> + ?tp(ocsp_refresh_timer, #{listener_id => ListenerID}), + ?SLOG(debug, #{ + msg => "refreshing_ocsp_response", + listener_id => ListenerID + }), + Conf = undefined, + handle_refresh(ListenerID, Conf, State0); + _ -> + {noreply, State0} + end; +handle_info({timeout, TRef, {refresh, ListenerID, Conf}}, State0) -> + case maps:get(?REFRESH_TIMER(ListenerID), State0, undefined) of + TRef -> + ?tp(ocsp_refresh_timer, #{listener_id => ListenerID}), + ?SLOG(debug, #{ + msg => "refreshing_ocsp_response", + listener_id => ListenerID + }), + handle_refresh(ListenerID, Conf, State0); + _ -> + {noreply, State0} + end; +handle_info(_Info, State) -> + {noreply, State}. + +code_change(_Vsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% internal functions +%%-------------------------------------------------------------------- + +http_fetch(ListenerID) -> + %% TODO: configurable call timeout? + gen_server:call(?MODULE, {http_fetch, ListenerID}, ?CALL_TIMEOUT). + +cache_key(ListenerID) -> + ?WITH_LISTENER_CONFIG( + ListenerID, + [ssl_options], + #{certfile := ServerCertPemPath}, + error, + begin + #'Certificate'{ + tbsCertificate = + #'TBSCertificate'{ + signature = Signature + } + } = read_server_cert(ServerCertPemPath), + {ok, {ocsp_response, Signature}} + end + ). + +do_lookup(ListenerID) -> + CacheKey = cache_key(ListenerID), + case CacheKey of + error -> + {error, invalid_listener_id}; + {ok, Key} -> + %% Respond immediately if a concurrent call already fetched it. + case ets:lookup(?CACHE_TAB, Key) of + [{_, DERResponse}] -> + ?tp(ocsp_cache_hit, #{listener_id => ListenerID}), + {ok, DERResponse}; + [] -> + {error, not_cached} + end + end. + +read_server_cert(ServerCertPemPath0) -> + ServerCertPemPath = to_bin(ServerCertPemPath0), + case ets:lookup(ssl_pem_cache, ServerCertPemPath) of + [{_, [{'Certificate', ServerCertDer, _} | _]}] -> + public_key:der_decode('Certificate', ServerCertDer); + [] -> + case file:read_file(ServerCertPemPath) of + {ok, ServerCertPem} -> + [{'Certificate', ServerCertDer, _} | _] = + public_key:pem_decode(ServerCertPem), + public_key:der_decode('Certificate', ServerCertDer); + {error, Error1} -> + error({bad_server_cert_file, Error1}) + end + end. + +handle_refresh(ListenerID, Conf, State0) -> + %% no point in retrying if the config is inconsistent or non + %% existent. + State1 = maps:without([{refresh_interval, ListenerID}, ?REFRESH_TIMER(ListenerID)], State0), + with_refresh_params(ListenerID, Conf, {noreply, State1}, fun(Params) -> + case do_http_fetch_and_cache(ListenerID, Params) of + error -> + ?SLOG(debug, #{ + msg => "failed_to_fetch_ocsp_response", + listener_id => ListenerID + }), + {noreply, ensure_timer(ListenerID, State0, ?RETRY_TIMEOUT)}; + {ok, _Response} -> + ?SLOG(debug, #{ + msg => "fetched_ocsp_response", + listener_id => ListenerID + }), + {noreply, ensure_timer(ListenerID, State0)} + end + end). + +with_refresh_params(ListenerID, Conf, ErrorRet, Fn) -> + case get_refresh_params(ListenerID, Conf) of + error -> + ErrorRet; + {ok, Params} -> + Fn(Params) + end. + +get_refresh_params(ListenerID, undefined = _Conf) -> + %% during normal periodic refreshes, we read from the emqx config. + ?WITH_LISTENER_CONFIG( + ListenerID, + [ssl_options], + #{ + ocsp := #{ + issuer_pem := IssuerPemPath, + responder_url := ResponderURL, + refresh_http_timeout := HTTPTimeout + }, + certfile := ServerCertPemPath + }, + error, + {ok, #{ + issuer_pem => IssuerPemPath, + responder_url => ResponderURL, + refresh_http_timeout => HTTPTimeout, + server_certfile => ServerCertPemPath + }} + ); +get_refresh_params(_ListenerID, #{ + ssl_options := #{ + ocsp := #{ + issuer_pem := IssuerPemPath, + responder_url := ResponderURL, + refresh_http_timeout := HTTPTimeout + }, + certfile := ServerCertPemPath + } +}) -> + {ok, #{ + issuer_pem => IssuerPemPath, + responder_url => ResponderURL, + refresh_http_timeout => HTTPTimeout, + server_certfile => ServerCertPemPath + }}; +get_refresh_params(_ListenerID, _Conf) -> + error. + +do_http_fetch_and_cache(ListenerID, Params) -> + #{ + issuer_pem := IssuerPemPath, + responder_url := ResponderURL, + refresh_http_timeout := HTTPTimeout, + server_certfile := ServerCertPemPath + } = Params, + IssuerPem = + case file:read_file(IssuerPemPath) of + {ok, IssuerPem0} -> IssuerPem0; + {error, Error0} -> error({bad_issuer_pem_file, Error0}) + end, + ServerCert = read_server_cert(ServerCertPemPath), + Request = build_ocsp_request(IssuerPem, ServerCert), + ?tp(ocsp_http_fetch, #{ + listener_id => ListenerID, + responder_url => ResponderURL, + timeout => HTTPTimeout + }), + RequestURI = iolist_to_binary([ResponderURL, Request]), + Resp = ?MODULE:http_get(RequestURI, HTTPTimeout), + case Resp of + {ok, {{_, 200, _}, _, Body}} -> + ?SLOG(debug, #{ + msg => "caching_ocsp_response", + listener_id => ListenerID + }), + %% if we got this far, the certfile is correct. + {ok, CacheKey} = cache_key(ListenerID), + true = ets:insert(?CACHE_TAB, {CacheKey, Body}), + ?tp(ocsp_http_fetch_and_cache, #{ + listener_id => ListenerID, + headers => true + }), + {ok, Body}; + {ok, {200, Body}} -> + ?SLOG(debug, #{ + msg => "caching_ocsp_response", + listener_id => ListenerID + }), + %% if we got this far, the certfile is correct. + {ok, CacheKey} = cache_key(ListenerID), + true = ets:insert(?CACHE_TAB, {CacheKey, Body}), + ?tp(ocsp_http_fetch_and_cache, #{ + listener_id => ListenerID, + headers => false + }), + {ok, Body}; + {ok, {{_, Code, _}, _, Body}} -> + ?tp( + error, + ocsp_http_fetch_bad_code, + #{ + listener_id => ListenerID, + body => Body, + code => Code, + headers => true + } + ), + ?SLOG(error, #{ + msg => "error_fetching_ocsp_response", + listener_id => ListenerID, + code => Code, + body => Body + }), + error; + {ok, {Code, Body}} -> + ?tp( + error, + ocsp_http_fetch_bad_code, + #{ + listener_id => ListenerID, + body => Body, + code => Code, + headers => false + } + ), + ?SLOG(error, #{ + msg => "error_fetching_ocsp_response", + listener_id => ListenerID, + code => Code, + body => Body + }), + error; + {error, Error} -> + ?tp( + error, + ocsp_http_fetch_error, + #{ + listener_id => ListenerID, + error => Error + } + ), + ?SLOG(error, #{ + msg => "error_fetching_ocsp_response", + listener_id => ListenerID, + error => Error + }), + error + end. + +http_get(URL, HTTPTimeout) -> + httpc:request( + get, + {URL, [{"connection", "close"}]}, + [{timeout, HTTPTimeout}], + [{body_format, binary}] + ). + +ensure_timer(ListenerID, State) -> + Timeout = maps:get({refresh_interval, ListenerID}, State, timer:minutes(5)), + ensure_timer(ListenerID, State, Timeout). + +ensure_timer(ListenerID, State, Timeout) -> + ensure_timer(ListenerID, {refresh, ListenerID}, State, Timeout). + +ensure_timer(ListenerID, Message, State, Timeout) -> + emqx_misc:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), + State#{ + ?REFRESH_TIMER(ListenerID) => emqx_misc:start_timer( + Timeout, + Message + ) + }. + +build_ocsp_request(IssuerPem, ServerCert) -> + [{'Certificate', IssuerDer, _} | _] = public_key:pem_decode(IssuerPem), + #'Certificate'{ + tbsCertificate = + #'TBSCertificate'{ + serialNumber = SerialNumber, + issuer = Issuer + } + } = ServerCert, + #'Certificate'{ + tbsCertificate = + #'TBSCertificate'{ + subjectPublicKeyInfo = + #'SubjectPublicKeyInfo'{subjectPublicKey = IssuerPublicKeyDer} + } + } = public_key:der_decode('Certificate', IssuerDer), + IssuerDNHash = crypto:hash(sha, public_key:der_encode('Name', Issuer)), + IssuerPKHash = crypto:hash(sha, IssuerPublicKeyDer), + Req = #'OCSPRequest'{ + tbsRequest = + #'TBSRequest'{ + version = 0, + requestList = + [ + #'Request'{ + reqCert = + #'CertID'{ + hashAlgorithm = + #'AlgorithmIdentifier'{ + algorithm = ?'id-sha1', + %% ??? + parameters = <<5, 0>> + }, + issuerNameHash = IssuerDNHash, + issuerKeyHash = IssuerPKHash, + serialNumber = SerialNumber + } + } + ] + } + }, + ReqDer = public_key:der_encode('OCSPRequest', Req), + base64:encode_to_string(ReqDer). + +to_bin(Str) when is_list(Str) -> list_to_binary(Str); +to_bin(Bin) when is_binary(Bin) -> Bin. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6f935f1e5..6412711a6 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -810,7 +810,7 @@ fields("mqtt_ssl_listener") -> {"ssl_options", sc( ref("listener_ssl_opts"), - #{} + #{validator => fun mqtt_ssl_listener_ssl_options_validator/1} )} ]; fields("mqtt_ws_listener") -> @@ -1294,6 +1294,56 @@ fields("listener_quic_ssl_opts") -> ); fields("ssl_client_opts") -> client_ssl_opts_schema(#{}); +fields("ocsp") -> + [ + {"enable_ocsp_stapling", + sc( + boolean(), + #{ + default => false, + desc => ?DESC("server_ssl_opts_schema_enable_ocsp_stapling") + } + )}, + {"responder_url", + sc( + binary(), + #{ + required => false, + validator => fun ocsp_responder_url_validator/1, + converter => fun + (undefined, _Opts) -> + undefined; + (URL, _Opts) -> + uri_string:normalize(URL) + end, + desc => ?DESC("server_ssl_opts_schema_ocsp_responder_url") + } + )}, + {"issuer_pem", + sc( + binary(), + #{ + required => false, + desc => ?DESC("server_ssl_opts_schema_ocsp_issuer_pem") + } + )}, + {"refresh_interval", + sc( + duration(), + #{ + default => <<"5m">>, + desc => ?DESC("server_ssl_opts_schema_ocsp_refresh_interval") + } + )}, + {"refresh_http_timeout", + sc( + duration(), + #{ + default => <<"15s">>, + desc => ?DESC("server_ssl_opts_schema_ocsp_refresh_http_timeout") + } + )} + ]; fields("deflate_opts") -> [ {"level", @@ -2017,6 +2067,8 @@ desc("trace") -> "Real-time filtering logs for the ClientID or Topic or IP for debugging."; desc("shared_subscription_group") -> "Per group dispatch strategy for shared subscription"; +desc("ocsp") -> + "Per listener OCSP Stapling configuration."; desc(_) -> undefined. @@ -2199,14 +2251,62 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> )} ] ++ [ - {"gc_after_handshake", - sc(boolean(), #{ - default => false, - desc => ?DESC(server_ssl_opts_schema_gc_after_handshake) - })} - || not IsRanchListener + Field + || not IsRanchListener, + Field <- [ + {"gc_after_handshake", + sc(boolean(), #{ + default => false, + desc => ?DESC(server_ssl_opts_schema_gc_after_handshake) + })}, + {"ocsp", + sc( + ref("ocsp"), + #{ + required => false, + validator => fun ocsp_inner_validator/1 + } + )} + ] ]. +mqtt_ssl_listener_ssl_options_validator(Conf) -> + Checks = [ + fun ocsp_outer_validator/1 + ], + case emqx_misc:pipeline(Checks, Conf, not_used) of + {ok, _, _} -> + ok; + {error, Reason, _NotUsed} -> + {error, Reason} + end. + +ocsp_outer_validator(#{<<"ocsp">> := #{<<"enable_ocsp_stapling">> := true}} = Conf) -> + %% outer mqtt listener ssl server config + ServerCertPemPath = maps:get(<<"certfile">>, Conf, undefined), + case ServerCertPemPath of + undefined -> + {error, "Server certificate must be defined when using OCSP stapling"}; + _ -> + %% check if issuer pem is readable and/or valid? + ok + end; +ocsp_outer_validator(_Conf) -> + ok. + +ocsp_inner_validator(#{enable_ocsp_stapling := _} = Conf) -> + ocsp_inner_validator(emqx_map_lib:binary_key_map(Conf)); +ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := false} = _Conf) -> + ok; +ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) -> + assert_required_field( + Conf, <<"responder_url">>, "The responder URL is required for OCSP stapling" + ), + assert_required_field( + Conf, <<"issuer_pem">>, "The issuer PEM path is required for OCSP stapling" + ), + ok. + %% @doc Make schema for SSL client. -spec client_ssl_opts_schema(map()) -> hocon_schema:field_schema(). client_ssl_opts_schema(Defaults) -> @@ -2865,3 +2965,11 @@ is_quic_ssl_opts(Name) -> %% , "handshake_timeout" %% , "gc_after_handshake" ]). + +assert_required_field(Conf, Key, ErrorMessage) -> + case maps:get(Key, Conf, undefined) of + undefined -> + throw(ErrorMessage); + _ -> + ok + end. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 8353c2895..c26e63a62 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -29,6 +29,7 @@ boot_modules/1, start_apps/1, start_apps/2, + start_apps/3, stop_apps/1, reload/2, app_path/2, @@ -36,7 +37,8 @@ deps_path/2, flush/0, flush/1, - render_and_load_app_config/1 + render_and_load_app_config/1, + render_and_load_app_config/2 ]). -export([ @@ -185,17 +187,21 @@ start_apps(Apps) -> application:set_env(system_monitor, db_hostname, ""), ok end, - start_apps(Apps, DefaultHandler). + start_apps(Apps, DefaultHandler, #{}). -spec start_apps(Apps :: apps(), Handler :: special_config_handler()) -> ok. start_apps(Apps, SpecAppConfig) when is_function(SpecAppConfig) -> + start_apps(Apps, SpecAppConfig, #{}). + +-spec start_apps(Apps :: apps(), Handler :: special_config_handler(), map()) -> ok. +start_apps(Apps, SpecAppConfig, Opts) when is_function(SpecAppConfig) -> %% Load all application code to beam vm first %% Because, minirest, ekka etc.. application will scan these modules lists:foreach(fun load/1, [emqx | Apps]), ok = start_ekka(), mnesia:clear_table(emqx_admin), ok = emqx_ratelimiter_SUITE:load_conf(), - lists:foreach(fun(App) -> start_app(App, SpecAppConfig) end, [emqx | Apps]). + lists:foreach(fun(App) -> start_app(App, SpecAppConfig, Opts) end, [emqx | Apps]). load(App) -> case application:load(App) of @@ -205,27 +211,31 @@ load(App) -> end. render_and_load_app_config(App) -> + render_and_load_app_config(App, #{}). + +render_and_load_app_config(App, Opts) -> load(App), Schema = app_schema(App), - Conf = app_path(App, filename:join(["etc", app_conf_file(App)])), + ConfFilePath = maps:get(conf_file_path, Opts, filename:join(["etc", app_conf_file(App)])), + Conf = app_path(App, ConfFilePath), try - do_render_app_config(App, Schema, Conf) + do_render_app_config(App, Schema, Conf, Opts) catch throw:E:St -> %% turn throw into error error({Conf, E, St}) end. -do_render_app_config(App, Schema, ConfigFile) -> - Vars = mustache_vars(App), +do_render_app_config(App, Schema, ConfigFile, Opts) -> + Vars = mustache_vars(App, Opts), RenderedConfigFile = render_config_file(ConfigFile, Vars), read_schema_configs(Schema, RenderedConfigFile), force_set_config_file_paths(App, [RenderedConfigFile]), copy_certs(App, RenderedConfigFile), ok. -start_app(App, SpecAppConfig) -> - render_and_load_app_config(App), +start_app(App, SpecAppConfig, Opts) -> + render_and_load_app_config(App, Opts), SpecAppConfig(App), case application:ensure_all_started(App) of {ok, _} -> @@ -248,12 +258,13 @@ app_schema(App) -> no_schema end. -mustache_vars(App) -> +mustache_vars(App, Opts) -> + ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, []), [ {platform_data_dir, app_path(App, "data")}, {platform_etc_dir, app_path(App, "etc")}, {platform_log_dir, app_path(App, "log")} - ]. + ] ++ ExtraMustacheVars. render_config_file(ConfigFile, Vars0) -> Temp = @@ -337,7 +348,7 @@ safe_relative_path_2(Path) -> -spec reload(App :: atom(), SpecAppConfig :: special_config_handler()) -> ok. reload(App, SpecAppConfigHandler) -> application:stop(App), - start_app(App, SpecAppConfigHandler), + start_app(App, SpecAppConfigHandler, #{}), application:start(App). ensure_mnesia_stopped() -> @@ -479,7 +490,7 @@ is_all_tcp_servers_available(Servers) -> {_, []} -> true; {_, Unavail} -> - ct:print("Unavailable servers: ~p", [Unavail]), + ct:pal("Unavailable servers: ~p", [Unavail]), false end. diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl new file mode 100644 index 000000000..1f5e34548 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -0,0 +1,908 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ocsp_cache_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include_lib("ssl/src/ssl_handshake.hrl"). + +-define(CACHE_TAB, emqx_ocsp_cache). + +all() -> + [{group, openssl}] ++ tests(). + +tests() -> + emqx_common_test_helpers:all(?MODULE) -- openssl_tests(). + +openssl_tests() -> + [t_openssl_client]. + +groups() -> + OpensslTests = openssl_tests(), + [ + {openssl, [ + {group, tls12}, + {group, tls13} + ]}, + {tls12, [ + {group, with_status_request}, + {group, without_status_request} + ]}, + {tls13, [ + {group, with_status_request}, + {group, without_status_request} + ]}, + {with_status_request, [], OpensslTests}, + {without_status_request, [], OpensslTests} + ]. + +init_per_suite(Config) -> + application:load(emqx), + emqx_config:save_schema_mod_and_names(emqx_schema), + emqx_common_test_helpers:boot_modules(all), + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(tls12, Config) -> + [{tls_vsn, "-tls1_2"} | Config]; +init_per_group(tls13, Config) -> + [{tls_vsn, "-tls1_3"} | Config]; +init_per_group(with_status_request, Config) -> + [{status_request, true} | Config]; +init_per_group(without_status_request, Config) -> + [{status_request, false} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(t_openssl_client, Config) -> + ct:timetrap({seconds, 30}), + DataDir = ?config(data_dir, Config), + Handler = fun(_) -> ok end, + {OCSPResponderPort, OCSPOSPid} = setup_openssl_ocsp(Config), + ConfFilePath = filename:join([DataDir, "openssl_listeners.conf"]), + emqx_common_test_helpers:start_apps( + [], + Handler, + #{ + extra_mustache_vars => [{test_data_dir, DataDir}], + conf_file_path => ConfFilePath + } + ), + ct:sleep(1_000), + [ + {ocsp_responder_port, OCSPResponderPort}, + {ocsp_responder_os_pid, OCSPOSPid} + | Config + ]; +init_per_testcase(TestCase, Config) when + TestCase =:= t_update_listener; + TestCase =:= t_validations +-> + %% when running emqx standalone tests, we can't use those + %% features. + case does_module_exist(emqx_mgmt_api_test_util) of + true -> + ct:timetrap({seconds, 30}), + %% start the listener with the default (non-ocsp) config + TestPid = self(), + ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), + meck:expect( + emqx_ocsp_cache, + http_get, + fun(URL, _HTTPTimeout) -> + ct:pal("ocsp http request ~p", [URL]), + TestPid ! {http_get, URL}, + {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} + end + ), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), + snabbkaffe:start_trace(), + Config; + false -> + [{skip_does_not_apply, true} | Config] + end; +init_per_testcase(t_ocsp_responder_error_responses, Config) -> + ct:timetrap({seconds, 30}), + TestPid = self(), + ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), + meck:expect( + emqx_ocsp_cache, + http_get, + fun(URL, _HTTPTimeout) -> + ct:pal("ocsp http request ~p", [URL]), + TestPid ! {http_get, URL}, + persistent_term:get({?MODULE, http_response}) + end + ), + DataDir = ?config(data_dir, Config), + Type = ssl, + Name = test_ocsp, + ListenerOpts = #{ + ssl_options => + #{ + certfile => filename:join(DataDir, "server.pem"), + ocsp => #{ + enable_ocsp_stapling => true, + responder_url => <<"http://localhost:9877/">>, + issuer_pem => filename:join(DataDir, "ocsp-issuer.pem"), + refresh_http_timeout => 15_000, + refresh_interval => 1_000 + } + } + }, + Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, + ConfBin = emqx_map_lib:binary_key_map(Conf), + hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), + emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), + snabbkaffe:start_trace(), + {ok, CachePid} = emqx_ocsp_cache:start_link(), + [ + {cache_pid, CachePid} + | Config + ]; +init_per_testcase(_TestCase, Config) -> + ct:timetrap({seconds, 10}), + TestPid = self(), + ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), + meck:expect( + emqx_ocsp_cache, + http_get, + fun(URL, _HTTPTimeout) -> + TestPid ! {http_get, URL}, + {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} + end + ), + {ok, CachePid} = emqx_ocsp_cache:start_link(), + DataDir = ?config(data_dir, Config), + Type = ssl, + Name = test_ocsp, + ListenerOpts = #{ + ssl_options => + #{ + certfile => filename:join(DataDir, "server.pem"), + ocsp => #{ + enable_ocsp_stapling => true, + responder_url => <<"http://localhost:9877/">>, + issuer_pem => filename:join(DataDir, "ocsp-issuer.pem"), + refresh_http_timeout => 15_000, + refresh_interval => 1_000 + } + } + }, + Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, + ConfBin = emqx_map_lib:binary_key_map(Conf), + hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), + emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), + snabbkaffe:start_trace(), + [ + {cache_pid, CachePid} + | Config + ]. + +end_per_testcase(t_openssl_client, Config) -> + OCSPResponderOSPid = ?config(ocsp_responder_os_pid, Config), + catch kill_pid(OCSPResponderOSPid), + emqx_common_test_helpers:stop_apps([]), + ok; +end_per_testcase(TestCase, Config) when + TestCase =:= t_update_listener; + TestCase =:= t_validations +-> + Skip = proplists:get_bool(skip_does_not_apply, Config), + case Skip of + true -> + ok; + false -> + emqx_mgmt_api_test_util:end_suite([emqx_conf]), + meck:unload([emqx_ocsp_cache]), + ok + end; +end_per_testcase(t_ocsp_responder_error_responses, Config) -> + CachePid = ?config(cache_pid, Config), + catch gen_server:stop(CachePid), + meck:unload([emqx_ocsp_cache]), + persistent_term:erase({?MODULE, http_response}), + ok; +end_per_testcase(_TestCase, Config) -> + CachePid = ?config(cache_pid, Config), + catch gen_server:stop(CachePid), + meck:unload([emqx_ocsp_cache]), + ok. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +does_module_exist(Mod) -> + case erlang:module_loaded(Mod) of + true -> + true; + false -> + case code:ensure_loaded(Mod) of + ok -> + true; + {module, Mod} -> + true; + _ -> + false + end + end. + +assert_no_http_get() -> + receive + {http_get, _URL} -> + error(should_be_cached) + after 0 -> + ok + end. + +assert_http_get(N) -> + assert_http_get(N, 0). + +assert_http_get(0, _Timeout) -> + ok; +assert_http_get(N, Timeout) when N > 0 -> + receive + {http_get, URL} -> + ?assertMatch(<<"http://localhost:9877/", _Request64/binary>>, URL), + ok + after Timeout -> + error({no_http_get, #{mailbox => process_info(self(), messages)}}) + end, + assert_http_get(N - 1, Timeout). + +spawn_openssl_client(TLSVsn, RequestStatus, Config) -> + DataDir = ?config(data_dir, Config), + ClientCert = filename:join([DataDir, "client.pem"]), + ClientKey = filename:join([DataDir, "client.key"]), + Cacert = filename:join([DataDir, "ca.pem"]), + Openssl = os:find_executable("openssl"), + StatusOpt = + case RequestStatus of + true -> ["-status"]; + false -> [] + end, + open_port( + {spawn_executable, Openssl}, + [ + {args, + [ + "s_client", + "-connect", + "localhost:8883", + %% needed to trigger `sni_fun' + "-servername", + "localhost", + TLSVsn, + "-CAfile", + Cacert, + "-cert", + ClientCert, + "-key", + ClientKey + ] ++ StatusOpt}, + binary, + stderr_to_stdout + ] + ). + +spawn_openssl_ocsp_responder(Config) -> + DataDir = ?config(data_dir, Config), + IssuerCert = filename:join([DataDir, "ocsp-issuer.pem"]), + IssuerKey = filename:join([DataDir, "ocsp-issuer.key"]), + Cacert = filename:join([DataDir, "ca.pem"]), + Index = filename:join([DataDir, "index.txt"]), + Openssl = os:find_executable("openssl"), + open_port( + {spawn_executable, Openssl}, + [ + {args, [ + "ocsp", + "-ignore_err", + "-port", + "9877", + "-CA", + Cacert, + "-rkey", + IssuerKey, + "-rsigner", + IssuerCert, + "-index", + Index + ]}, + binary, + stderr_to_stdout + ] + ). + +kill_pid(OSPid) -> + os:cmd("kill -9 " ++ integer_to_list(OSPid)). + +test_ocsp_connection(TLSVsn, WithRequestStatus = true, Config) -> + ClientPort = spawn_openssl_client(TLSVsn, WithRequestStatus, Config), + {os_pid, ClientOSPid} = erlang:port_info(ClientPort, os_pid), + try + timer:sleep(timer:seconds(1)), + {messages, Messages} = process_info(self(), messages), + OCSPOutput0 = [ + Output + || {_Port, {data, Output}} <- Messages, + re:run(Output, "OCSP response:") =/= nomatch + ], + ?assertMatch( + [_], + OCSPOutput0, + #{all_messages => Messages} + ), + [OCSPOutput] = OCSPOutput0, + ?assertMatch( + {match, _}, + re:run(OCSPOutput, "OCSP Response Status: successful"), + #{all_messages => Messages} + ), + ?assertMatch( + {match, _}, + re:run(OCSPOutput, "Cert Status: good"), + #{all_messages => Messages} + ), + ok + after + catch kill_pid(ClientOSPid) + end; +test_ocsp_connection(TLSVsn, WithRequestStatus = false, Config) -> + ClientPort = spawn_openssl_client(TLSVsn, WithRequestStatus, Config), + {os_pid, ClientOSPid} = erlang:port_info(ClientPort, os_pid), + try + timer:sleep(timer:seconds(1)), + {messages, Messages} = process_info(self(), messages), + OCSPOutput = [ + Output + || {_Port, {data, Output}} <- Messages, + re:run(Output, "OCSP response:") =/= nomatch + ], + ?assertEqual( + [], + OCSPOutput, + #{all_messages => Messages} + ), + ok + after + catch kill_pid(ClientOSPid) + end. + +ensure_port_open(Port) -> + do_ensure_port_open(Port, 10). + +do_ensure_port_open(Port, 0) -> + error({port_not_open, Port}); +do_ensure_port_open(Port, N) when N > 0 -> + Timeout = 1_000, + case gen_tcp:connect("localhost", Port, [], Timeout) of + {ok, Sock} -> + gen_tcp:close(Sock), + ok; + {error, _} -> + ct:sleep(500), + do_ensure_port_open(Port, N - 1) + end. + +get_sni_fun(ListenerID) -> + #{opts := Opts} = emqx_listeners:find_by_id(ListenerID), + SSLOpts = proplists:get_value(ssl_options, Opts), + proplists:get_value(sni_fun, SSLOpts). + +openssl_version() -> + Res0 = string:trim(os:cmd("openssl version"), trailing), + [_, Res] = string:split(Res0, " "), + {match, [Version]} = re:run(Res, "^([^ ]+)", [{capture, first, list}]), + Version. + +setup_openssl_ocsp(Config) -> + OCSPResponderPort = spawn_openssl_ocsp_responder(Config), + {os_pid, OCSPOSPid} = erlang:port_info(OCSPResponderPort, os_pid), + %%%%%%%% Warning!!! + %% Apparently, openssl 3.0.7 introduced a bug in the responder + %% that makes it hang forever if one probes the port with + %% `gen_tcp:open' / `gen_tcp:close'... Comment this out if + %% openssl gets updated in CI or in your local machine. + OpenSSLVersion = openssl_version(), + ct:pal("openssl version: ~p", [OpenSSLVersion]), + case OpenSSLVersion of + "3." ++ _ -> + %% hope that the responder has started... + ok; + _ -> + ensure_port_open(9877) + end, + ct:sleep(1_000), + {OCSPResponderPort, OCSPOSPid}. + +request(Method, Url, QueryParams, Body) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of + {ok, {Reason, Headers, BodyR}} -> + {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + Error -> + Error + end. + +get_listener_via_api(ListenerId) -> + Path = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), + request(get, Path, [], []). + +update_listener_via_api(ListenerId, NewConfig) -> + Path = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), + request(put, Path, [], NewConfig). + +put_http_response(Response) -> + persistent_term:put({?MODULE, http_response}, Response). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_request_ocsp_response(_Config) -> + ?check_trace( + begin + ListenerID = <<"ssl:test_ocsp">>, + %% not yet cached. + ?assertEqual([], ets:tab2list(?CACHE_TAB)), + ?assertEqual( + {ok, <<"ocsp response">>}, + emqx_ocsp_cache:fetch_response(ListenerID) + ), + assert_http_get(1), + ?assertMatch([{_, <<"ocsp response">>}], ets:tab2list(?CACHE_TAB)), + %% already cached; should not perform request again. + ?assertEqual( + {ok, <<"ocsp response">>}, + emqx_ocsp_cache:fetch_response(ListenerID) + ), + assert_no_http_get(), + ok + end, + fun(Trace) -> + ?assert( + ?strict_causality( + #{?snk_kind := ocsp_cache_miss, listener_id := _ListenerID}, + #{?snk_kind := ocsp_http_fetch_and_cache, listener_id := _ListenerID}, + Trace + ) + ), + ?assertMatch( + [_], + ?of_kind(ocsp_cache_miss, Trace) + ), + ?assertMatch( + [_], + ?of_kind(ocsp_http_fetch_and_cache, Trace) + ), + ?assertMatch( + [_], + ?of_kind(ocsp_cache_hit, Trace) + ), + ok + end + ). + +t_request_ocsp_response_restart_cache(Config) -> + process_flag(trap_exit, true), + CachePid = ?config(cache_pid, Config), + ListenerID = <<"ssl:test_ocsp">>, + ?check_trace( + begin + [] = ets:tab2list(?CACHE_TAB), + {ok, _} = emqx_ocsp_cache:fetch_response(ListenerID), + ?wait_async_action( + begin + Ref = monitor(process, CachePid), + exit(CachePid, kill), + receive + {'DOWN', Ref, process, CachePid, killed} -> + ok + after 1_000 -> + error(cache_not_killed) + end, + {ok, _} = emqx_ocsp_cache:start_link(), + ok + end, + #{?snk_kind := ocsp_cache_init} + ), + {ok, _} = emqx_ocsp_cache:fetch_response(ListenerID), + ok + end, + fun(Trace) -> + ?assertMatch( + [_, _], + ?of_kind(ocsp_http_fetch_and_cache, Trace) + ), + assert_http_get(2), + ok + end + ). + +t_request_ocsp_response_bad_http_status(_Config) -> + TestPid = self(), + meck:expect( + emqx_ocsp_cache, + http_get, + fun(URL, _HTTPTimeout) -> + TestPid ! {http_get, URL}, + {ok, {{"HTTP/1.0", 404, 'Not Found'}, [], <<"not found">>}} + end + ), + ListenerID = <<"ssl:test_ocsp">>, + %% not yet cached. + ?assertEqual([], ets:tab2list(?CACHE_TAB)), + ?assertEqual( + error, + emqx_ocsp_cache:fetch_response(ListenerID) + ), + assert_http_get(1), + ?assertEqual([], ets:tab2list(?CACHE_TAB)), + ok. + +t_request_ocsp_response_timeout(_Config) -> + TestPid = self(), + meck:expect( + emqx_ocsp_cache, + http_get, + fun(URL, _HTTPTimeout) -> + TestPid ! {http_get, URL}, + {error, timeout} + end + ), + ListenerID = <<"ssl:test_ocsp">>, + %% not yet cached. + ?assertEqual([], ets:tab2list(?CACHE_TAB)), + ?assertEqual( + error, + emqx_ocsp_cache:fetch_response(ListenerID) + ), + assert_http_get(1), + ?assertEqual([], ets:tab2list(?CACHE_TAB)), + ok. + +t_register_listener(_Config) -> + ListenerID = <<"ssl:test_ocsp">>, + Conf = emqx_config:get_listener_conf(ssl, test_ocsp, []), + %% should fetch and cache immediately + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerID, Conf), + #{?snk_kind := ocsp_http_fetch_and_cache, listener_id := ListenerID} + ), + assert_http_get(1), + ?assertMatch([{_, <<"ocsp response">>}], ets:tab2list(?CACHE_TAB)), + ok. + +t_register_twice(_Config) -> + ListenerID = <<"ssl:test_ocsp">>, + Conf = emqx_config:get_listener_conf(ssl, test_ocsp, []), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerID, Conf), + #{?snk_kind := ocsp_http_fetch_and_cache, listener_id := ListenerID} + ), + assert_http_get(1), + ?assertMatch([{_, <<"ocsp response">>}], ets:tab2list(?CACHE_TAB)), + %% should have no problem in registering the same listener again. + %% this prompts an immediate refresh. + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerID, Conf), + #{?snk_kind := ocsp_http_fetch_and_cache, listener_id := ListenerID} + ), + ok. + +t_refresh_periodically(_Config) -> + ListenerID = <<"ssl:test_ocsp">>, + Conf = emqx_config:get_listener_conf(ssl, test_ocsp, []), + %% should refresh periodically + {ok, SubRef} = + snabbkaffe:subscribe( + fun + (#{?snk_kind := ocsp_http_fetch_and_cache, listener_id := ListenerID0}) -> + ListenerID0 =:= ListenerID; + (_) -> + false + end, + _NEvents = 2, + _Timeout = 10_000 + ), + ok = emqx_ocsp_cache:register_listener(ListenerID, Conf), + ?assertMatch({ok, [_, _]}, snabbkaffe:receive_events(SubRef)), + assert_http_get(2), + ok. + +t_sni_fun_success(_Config) -> + ListenerID = <<"ssl:test_ocsp">>, + ServerName = "localhost", + ?assertEqual( + [ + {certificate_status, #certificate_status{ + status_type = ?CERTIFICATE_STATUS_TYPE_OCSP, + response = <<"ocsp response">> + }} + ], + emqx_ocsp_cache:sni_fun(ServerName, ListenerID) + ), + ok. + +t_sni_fun_http_error(_Config) -> + meck:expect( + emqx_ocsp_cache, + http_get, + fun(_URL, _HTTPTimeout) -> + {error, timeout} + end + ), + ListenerID = <<"ssl:test_ocsp">>, + ServerName = "localhost", + ?assertEqual( + [], + emqx_ocsp_cache:sni_fun(ServerName, ListenerID) + ), + ok. + +%% check that we can start with a non-ocsp stapling listener and +%% restart it with the new ocsp config. +t_update_listener(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + do_t_update_listener(Config) + end. + +do_t_update_listener(Config) -> + DataDir = ?config(data_dir, Config), + Keyfile = filename:join([DataDir, "server.key"]), + Certfile = filename:join([DataDir, "server.pem"]), + Cacertfile = filename:join([DataDir, "ca.pem"]), + IssuerPem = filename:join([DataDir, "ocsp-issuer.pem"]), + + %% no ocsp at first + ListenerId = "ssl:default", + {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), + ?assertMatch( + #{ + <<"ssl_options">> := + #{ + <<"ocsp">> := + #{<<"enable_ocsp_stapling">> := false} + } + }, + ListenerData0 + ), + assert_no_http_get(), + + %% configure ocsp + OCSPConfig = + #{ + <<"ssl_options">> => + #{ + <<"keyfile">> => Keyfile, + <<"certfile">> => Certfile, + <<"cacertfile">> => Cacertfile, + <<"ocsp">> => + #{ + <<"enable_ocsp_stapling">> => true, + <<"issuer_pem">> => IssuerPem, + <<"responder_url">> => <<"http://localhost:9877">> + } + } + }, + ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, OCSPConfig), + {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), + ?assertMatch( + #{ + <<"ssl_options">> := + #{ + <<"ocsp">> := + #{ + <<"enable_ocsp_stapling">> := true, + <<"issuer_pem">> := _, + <<"responder_url">> := _ + } + } + }, + ListenerData2 + ), + assert_http_get(1, 5_000), + ok. + +t_ocsp_responder_error_responses(_Config) -> + ListenerId = <<"ssl:test_ocsp">>, + Conf = emqx_config:get_listener_conf(ssl, test_ocsp, []), + ?check_trace( + begin + %% successful response without headers + put_http_response({ok, {200, <<"ocsp_response">>}}), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerId, Conf), + #{?snk_kind := ocsp_http_fetch_and_cache, headers := false}, + 1_000 + ), + + %% error response with headers + put_http_response({ok, {{"HTTP/1.0", 500, "Internal Server Error"}, [], <<"error">>}}), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerId, Conf), + #{?snk_kind := ocsp_http_fetch_bad_code, code := 500, headers := true}, + 1_000 + ), + + %% error response without headers + put_http_response({ok, {500, <<"error">>}}), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerId, Conf), + #{?snk_kind := ocsp_http_fetch_bad_code, code := 500, headers := false}, + 1_000 + ), + + %% econnrefused + put_http_response( + {error, + {failed_connect, [ + {to_address, {"localhost", 9877}}, + {inet, [inet], econnrefused} + ]}} + ), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerId, Conf), + #{?snk_kind := ocsp_http_fetch_error, error := {failed_connect, _}}, + 1_000 + ), + + %% timeout + put_http_response({error, timeout}), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerId, Conf), + #{?snk_kind := ocsp_http_fetch_error, error := timeout}, + 1_000 + ), + + ok + end, + [] + ), + ok. + +t_unknown_requests(_Config) -> + emqx_ocsp_cache ! unknown, + ?assertEqual(ok, gen_server:cast(emqx_ocsp_cache, unknown)), + ?assertEqual({error, {unknown_call, unknown}}, gen_server:call(emqx_ocsp_cache, unknown)), + ok. + +t_validations(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + do_t_validations(Config) + end. + +do_t_validations(_Config) -> + ListenerId = <<"ssl:default">>, + {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), + + ListenerData1 = + emqx_map_lib:deep_merge( + ListenerData0, + #{ + <<"ssl_options">> => + #{<<"ocsp">> => #{<<"enable_ocsp_stapling">> => true}} + } + ), + {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), + #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = + emqx_json:decode(ResRaw1, [return_maps]), + ?assertMatch( + #{ + <<"mismatches">> := + #{ + <<"listeners:ssl_not_required_bind">> := + #{ + <<"reason">> := + <<"The responder URL is required for OCSP stapling">> + } + } + }, + emqx_json:decode(MsgRaw1, [return_maps]) + ), + + ListenerData2 = + emqx_map_lib:deep_merge( + ListenerData0, + #{ + <<"ssl_options">> => + #{ + <<"ocsp">> => #{ + <<"enable_ocsp_stapling">> => true, + <<"responder_url">> => <<"http://localhost:9877">> + } + } + } + ), + {error, {_, _, ResRaw2}} = update_listener_via_api(ListenerId, ListenerData2), + #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw2} = + emqx_json:decode(ResRaw2, [return_maps]), + ?assertMatch( + #{ + <<"mismatches">> := + #{ + <<"listeners:ssl_not_required_bind">> := + #{ + <<"reason">> := + <<"The issuer PEM path is required for OCSP stapling">> + } + } + }, + emqx_json:decode(MsgRaw2, [return_maps]) + ), + + ListenerData3a = + emqx_map_lib:deep_merge( + ListenerData0, + #{ + <<"ssl_options">> => + #{ + <<"ocsp">> => #{ + <<"enable_ocsp_stapling">> => true, + <<"responder_url">> => <<"http://localhost:9877">>, + <<"issuer_pem">> => <<"some_file">> + } + } + } + ), + ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), + {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), + #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = + emqx_json:decode(ResRaw3, [return_maps]), + ?assertMatch( + #{ + <<"mismatches">> := + #{ + <<"listeners:ssl_not_required_bind">> := + #{ + <<"reason">> := + <<"Server certificate must be defined when using OCSP stapling">> + } + } + }, + emqx_json:decode(MsgRaw3, [return_maps]) + ), + + ok. + +t_openssl_client(Config) -> + TLSVsn = ?config(tls_vsn, Config), + WithStatusRequest = ?config(status_request, Config), + %% ensure ocsp response is already cached. + ListenerID = <<"ssl:default">>, + ?assertMatch( + {ok, _}, + emqx_ocsp_cache:fetch_response(ListenerID), + #{msgs => process_info(self(), messages)} + ), + timer:sleep(500), + test_ocsp_connection(TLSVsn, WithStatusRequest, Config). diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ca.pem b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ca.pem new file mode 100644 index 000000000..eaabd2445 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ca.pem @@ -0,0 +1,68 @@ +-----BEGIN CERTIFICATE----- +MIIF+zCCA+OgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwbzELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQK +DAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENBMREwDwYDVQQDDAhNeVJvb3RD +QTAeFw0yMzAxMTIxMzA4MTZaFw0zMzAxMDkxMzA4MTZaMGsxCzAJBgNVBAYTAlNF +MRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAoMCU15T3JnTmFtZTEZMBcGA1UE +CwwQTXlJbnRlcm1lZGlhdGVDQTEZMBcGA1UEAwwQTXlJbnRlcm1lZGlhdGVDQTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALQG7dMeU/y9HDNHzhydR0bm +wN9UGplqJOJPwqJRaZZcrn9umgJ9SU2il2ceEVxMDwzBWCRKJO5/H9A9k13SqsXM +2c2c9xXfIF1kb820lCm1Uow5hZ/auDjxliNk9kNJDigCRi3QoIs/dVeWzFsgEC2l +gxRqauN2eNFb6/yXY788YALHBsCRV2NFOFXxtPsvLXpD9Q/8EqYsSMuLARRdHVNU +ryaEF5lhShpcuz0TlIuTy2TiuXJUtJ+p7a4Z7friZ6JsrmQWsVQBj44F8TJRHWzW +C7vm9c+dzEX9eqbr5iPL+L4ctMW9Lz6ePcYfIXne6CElusRUf8G+xM1uwovF9bpV ++9IqY7tAu9G1iY9iNtJgNNDKOCcOGKcZCx6Cg1XYOEKReNnUMazvYeqRrrjV5WQ0 +vOcD5zcBRNTXCddCLa7U0guXP9mQrfuk4NTH1Bt77JieTJ8cfDXHwtaKf6aGbmZP +wl1Xi/GuXNUP/xeog78RKyFwBmjt2JKwvWzMpfmH4mEkG9moh2alva+aEz6LIJuP +16g6s0Q6c793/OvUtpNcewHw4Vjn39LD9o6VLp854G4n8dVpUWSbWS+sXD1ZE69H +g/sMNMyq+09ufkbewY8xoCm/rQ1pqDZAVMWsstJEaYu7b/eb7R+RGOj1YECCV/Yp +EZPdDotbSNRkIi2d/a1NAgMBAAGjgaQwgaEwHQYDVR0OBBYEFExwhjsVUom6tQ+S +qq6xMUETvnPzMB8GA1UdIwQYMBaAFD90kfU5pc5l48THu0Ayj9SNpHuhMBIGA1Ud +EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDsGA1UdHwQ0MDIwMKAuoCyG +Kmh0dHA6Ly9sb2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUuY3JsLnBlbTANBgkq +hkiG9w0BAQsFAAOCAgEAK6NgdWQYtPNKQNBGjsgtgqTRh+k30iqSO6Y3yE1KGABO +EuQdVqkC2qUIbCB0M0qoV0ab50KNLfU6cbshggW4LDpcMpoQpI05fukNh1jm3ZuZ +0xsB7vlmlsv00tpqmfIl/zykPDynHKOmFh/hJP/KetMy4+wDv4/+xP31UdEj5XvG +HvMtuqOS23A+H6WPU7ol7KzKBnU2zz/xekvPbUD3JqV+ynP5bgbIZHAndd0o9T8e +NFX23Us4cTenU2/ZlOq694bRzGaK+n3Ksz995Nbtzv5fbUgqmf7Mcq4iHGRVtV11 +MRyBrsXZp2vbF63c4hrf2Zd6SWRoaDKRhP2DMhajpH9zZASSTlfejg/ZRO2s+Clh +YrSTkeMAdnRt6i/q4QRcOTCfsX75RFM5v67njvTXsSaSTnAwaPi78tRtf+WSh0EP +VVPzy++BszBVlJ1VAf7soWZHCjZxZ8ZPqVTy5okoHwWQ09WmYe8GfulDh1oj0wbK +3FjN7bODWHJN+bFf5aQfK+tumYKoPG8RXL6QxpEzjFWjxhIMJHHMKfDWnAV1o1+7 +/1/aDzq7MzEYBbrgQR7oE5ZHtyqhCf9LUgw0Kr7/8QWuNAdeDCJzjXRROU0hJczp +dOyfRlLbHmLLmGOnROlx6LsGNQ17zuz6SPi7ei8/ylhykawDOAGkM1+xFakmQhM= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUYjc7hD7/UJ0/VPADfNfp/WpOwRowDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCU0UxEjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJ +U3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENB +MREwDwYDVQQDDAhNeVJvb3RDQTAeFw0yMzAxMTIxMzA4MTRaFw00MzAxMDcxMzA4 +MTRaMG8xCzAJBgNVBAYTAlNFMRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcM +CVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMREwDwYDVQQLDAhNeVJvb3RD +QTERMA8GA1UEAwwITXlSb290Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCnBwSOYVJw47IoMHMXTVDtOYvUt3rqsurEhFcB4O8xmf2mmwr6m7s8A5Ft +AvAehg1GvnXT3t/KiyU7BK+acTwcErGyZwS2wvdB0lpHWSpOn/u5y+4ZETvQefcj +ZTdDOM9VN5nutpitgNb+1yL8sqSexfVbY7DnYYvFjOVBYoP/SGvM9jVjCad+0WL3 +FhuD+L8QAxzCieX3n9UMymlFwINQuEc+TDjuNcEqt+0J5EgS1fwzxb2RCVL0TNv4 +9a71hFGCNRj20AeZm99hbdufm7+0AFO7ocV5q43rLrWFUoBzqKPYIjga/cv/UdWZ +c5RLRXw3JDSrCqkf/mOlaEhNPlmWRF9MSus5Da3wuwgGCaVzmrf30rWR5aHHcscG +e+AOgJ4HayvBUQeb6ZlRXc0YlACiLToMKxuyxDyUcDfVEXpUIsDILF8dkiVQxEU3 +j9g6qjXiqPVdNiwpqXfBKObj8vNCzORnoHYs8cCgib3RgDVWeqkDmlSwlZE7CvQh +U4Loj4l7813xxzYEKkVaT1JdXPWu42CG/b4Y/+f4V+3rkJkYzUwndX6kZNksIBai +phmtvKt+CTdP1eAbT+C9AWWF3PT31+BIhuT0u9tR8BVSkXdQB8dG4M/AAJcTo640 +0mdYYOXT153gEKHJuUBm750ZTy+r6NjNvpw8VrMAakJwHqnIdQIDAQABo2MwYTAd +BgNVHQ4EFgQUP3SR9TmlzmXjxMe7QDKP1I2ke6EwHwYDVR0jBBgwFoAUP3SR9Tml +zmXjxMe7QDKP1I2ke6EwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYw +DQYJKoZIhvcNAQELBQADggIBAFMFv4C+I0+xOAb9v6G/IOpfPBZ1ez31EXKJJBra +lulP4nRHQMeb310JS8BIeQ3dl+7+PkSxPABZSwc3jkxdSMvhc+Z4MQtTgos+Qsjs +gH7sTqwWeeQ0lHYxWmkXijrh5OPRZwTKzYQlkcn85BCUXl2KDuNEdiqPbDTao+lc +lA0/UAvC6NCyFKq/jqf4CmW5Kx6yG1v1LaE+IXn7cbIXj+DaehocVXi0wsXqj03Q +DDUHuLHZP+LBsg4e91/0Jy2ekNRTYJifSqr+9ufHl0ZX1pFDZyf396IgZ5CQZ0PJ +nRxZHlCfsxWxmxxdy3FQSE6YwXhdTjjoAa1ApZcKkkt1beJa6/oRLze/ux5x+5q+ +4QczufHd6rjoKBi6BM3FgFQ8As5iNohHXlMHd/xITo1Go3CWw2j9TGH5vzksOElK +B0mcwwt2zwNEjvfytc+tI5jcfGN3tiT5fVHS8hw9dWKevypLL+55Ua9G8ZgDHasT +XFRJHgmnbyFcaAe26D2dSKmhC9u2mHBH+MaI8dj3e7wNBfpxNgp41aFIk+QTmiFW +VXFED6DHQ/Mxq93ACalHdYg18PlIYClbT6Pf2xXBnn33YPhn5xzoTZ+cDH/RpaQp +s0UUTSJT1UTXgtXPnZWQfvKlMjJEIiVFiLEC0sgZRlWuZDRAY0CdZJJxvQp59lqu +cbTm +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.key b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.key new file mode 100644 index 000000000..a1c46aa5c --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCmfZmBAOZJ8xjP +YkpyQxTGZ40vIwOuylwSow12idWN6jcW9g5aIip+B2oKrfzR7PYsxbDodcj/KOpQ +GwCFAujSYgYviiOsmATQ1meNocnnWjAsybw+dSXK/ZjfrVgIaJF7RHaLiDtq5TI4 +b4KjUFyh5NILIc+zfZqoNU6khUF0bcOBAG2BFaBzRf+a/hgZXEPyEnoqFK5J5k+D +DSlKXDbOTEHhXG4QFT1hZataxptD1nTEFRYuzfmh/g4RDvWtawm9YU3j/V0Un7t/ +Taj0fAXNi30TzKOVaVcDrkVtDFHe2hX3lOJd53I5NpS7asaq+aTNytz+I3Bf/a4v +khEgrKpjBSXXm/+Vw5NzsXNwKddSUGywmIbV2YBYnK+0DwhOXLsTPh3pv6931NVx +pifW0nM4Ur6XCDHOPVX/jIZZ819bzAlZZ3BgMTz7pqT9906lmNRQBgSgr+Zaw9gj +VhLg1VDfwF85eanhbzk5ITnffR+s2conZr2g+LEDsq2dJv/sEbYuHBNBkDthn439 +MgNq1nr3PV0hn8pNcgS5ZFUw+fN8403RY9TYLssB/FFYREDCax0j75qL3E7LbZK8 +JfsP8uh1e3PdR64TgtoYoTKuwtIqelmh+ryAWFjaXLPoP/AqYk1VcRCevOXUKw6L +iskdukplk9cy2cPLcm+EP+2Js3B28QIDAQABAoICABxBnVOcZjk/QaLy1N07HtPE +f9zz5Zxc+k7sbuzDHGQzT8m9FXb9LPaKRhhNaqbrP2WeYLW3RdduZ4QUbRxl/8Mz +AUdAu+i/PTP/a4BJaOWztBDp5SG5iqI+s5skxZfZvXUtC6yHQMRV5VXYMRUMHsiY +OADNKn3VT7IEKBZ6ij8bIO7sNmmN1NczllvFC6yEMQDs22B4dZMTvENq8KrO5ztQ +jG7V29Utcact1Oz5X6EeDN+5j3P+n8M7RcJl5lLaI4NJeCl9VvaY3H7Q3J+vy+FU +bvQ1Cz9gqzSz91L4YA3BODC2i0uyK/vjVE9Roimi6HJH34VfWONlv9IRiYgg3eLd +xrWe/qZkxcfrHmgyH0a6fxwpT58T3d6WH0I/HwSbJuVvm2AhLy+7zXdLNRLrlE+n +UfrJDgTwiTPlJA5JzSVGKVBSOVQs9G52aZ0IAvgN9uHHFhhqeJ3naax1q/JtRfDo +O0w5Ga2KjAJDcAQj/Cq5+LMSI1Bxl46db17EFnA//X3Oxhv93CvsTULPiOJ7fdYC +3X7YCJ33a7w4B8+FxmiTYLe+aR6CC8fsu4qYccCctPUje1MzUkw6gvbWSyxkbmW7 +kGTWKx4E/SL4cc+DjoC1h37RtqghDDxtYhA42wWiocDXoKPlWJoIkG1UUO5f6/2N +cKPzQx1f23UTvIRkMYe1AoIBAQDR94YzLncfuY4DhHpqJRjv8xXfOif+ARWicnma +CwePpv80YoQvc7B9rbPA9qZ5EG9eQF62FkTrvCwbAhA5L11aJsXxnSvZREQcdteO +kQPnKXAJbHYh5yto/HhezdtIMmoZCGpHLmsiK20QnRyA0InKsFCKBpi20gFzOKMx +DwuQEoANHIwUscHnansM958eKAolujfjjOeFiK+j4Vd6P0neV8EQTl6A0+R/l5td +l69wySW7tB4xfOon5Y0D+AfGMH3alZs3ymAjBNKZIk+2hKvhDRa7IqwlckwQq6by +Ku25LKeRVt3wOkfJitSDgiEsNA5oJQ90A4ny6hIOAvLWir6tAoIBAQDK/fPVaT7r +7tNjzaMgeQ/VKGUauCMbPC7ST2cEvZMp9YFhdKbl/TwhC8lpJqrsKhXyKNz20FOL +7m8XjHu4mdSs6zaPvkMnUboge9pcnIKeS5nRVsW0CRuSc4A3qhrvBp9av77gIjnr +XJ6RyFihDji1P6RVoylyyR8k/qiZupMg7UK3vbuTpJqARObfaaprOwqVItkJX2vf +XF7qfBCnik1jlZKWZq+9dbhz8KP4KWpKINrwIuvlAQnTJpc15beHxMEt73hxAY3A +n3Iydtm5zsBcOLyLLgySUOsp0zlcAv0iHP3ShsFP2WeQLKR9Qapc58kkJ1lmlu71 +QdahwonpXjXVAoIBAEQnfYc1iPNiTsezg+zad9rDZBEeloaroXMmh3RKKj0l7ub5 +J4Ejo2FYNeXn6ieX/x5v9I5UcjC21vY5WDzHtBykQ1JnOyl+MEGxDc04IzUwzS4x +57KfkAa3FPdpCMnJm4jeo2jRl3Ly96cR6IOjrWZ+jtYOyBln15KoCsjM4mr0pl4b +Kxk4jgFpHeIaqqqmQoz2gle5kBlXQfQHHFcRHhAvGfsKBUD6Bsyn0IWzy/3nPPlN +wRM9QeCLcZedNiDN8rw2HbkhVs1nLlkIuyk6rXQSxJMf8RMCo9Axd7JZ3uphpU7X +DJmCwXSZPNwnLE9l4ltJ1FdLIscX1Z54tIyRYs0CggEBAIVPgnMFS21myy0gP6Fz +4BH9FWkWxPd97sHvo5hZZ+yGbxGxqmoghPyu4PdNjbLLcN44N+Vfq36aeBrfB+GU +JTfqwUpliXSpF7N9o0pu/tk2jS4N7ojt8k2bzPjBni6cCstuYcyQrbkEep8DFDGx +RUzDHwmevfnEW8/P7qoG/dkB+G7zC91KnKzgkz7mBiWmAK0w1ZhyMkXeQ/d6wvVE +vs5HzJ05kvC5/wklYIn5qPRF34MVbBZZODqTfXrIAmAHt1aTjmWov49hJ348z4BX +Z70pBanh9B+jRM2TCniC/fsJTyiTlyD5hioJJ32bQmcBUfeMYAof1Y78ThityiSY +2oECggEAYdkz6z+1hIMI2nIMtei1n5bLV4bWmS1nkZ3pBSMkbS7VJFAxZ53xJi0S +StSs/bka+akvnYEoFAGhVtiaz4497qnUiquf/aBs4TUHfNGn22/LN5b8vs51ugil +RXejaJjPLqL6jmXz5T4+TJGcH5kL6NDtYkT3IEtv5uWkQkBs0Z1Juf34nVjMbozC +bohyOyCMOLt7HqcUpUtevSK7SXmyU4yd2UyRqFMFPi4RJjxQWFZmNFC5S1PsZBh+ +OOMNAJ1F2h2fC7KdNVBpdoNsOAPxdCNxbwGKiNHwnukvF9uvaDIw3jqKJU3g/Z6j +rkE8Bz5a/iwO+QwdO5Q2cp5+0nm41A== +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.pem b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.pem new file mode 100644 index 000000000..06adc2aa3 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/client.pem @@ -0,0 +1,38 @@ +-----BEGIN CERTIFICATE----- +MIIGmjCCBIKgAwIBAgICEAYwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDMwNjE5NTA0N1oXDTMzMDYxMTE5NTA0N1owezELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExFTATBgNVBAMMDG9j +c3AuY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKZ9mYEA +5knzGM9iSnJDFMZnjS8jA67KXBKjDXaJ1Y3qNxb2DloiKn4Hagqt/NHs9izFsOh1 +yP8o6lAbAIUC6NJiBi+KI6yYBNDWZ42hyedaMCzJvD51Jcr9mN+tWAhokXtEdouI +O2rlMjhvgqNQXKHk0gshz7N9mqg1TqSFQXRtw4EAbYEVoHNF/5r+GBlcQ/ISeioU +rknmT4MNKUpcNs5MQeFcbhAVPWFlq1rGm0PWdMQVFi7N+aH+DhEO9a1rCb1hTeP9 +XRSfu39NqPR8Bc2LfRPMo5VpVwOuRW0MUd7aFfeU4l3ncjk2lLtqxqr5pM3K3P4j +cF/9ri+SESCsqmMFJdeb/5XDk3Oxc3Ap11JQbLCYhtXZgFicr7QPCE5cuxM+Hem/ +r3fU1XGmJ9bSczhSvpcIMc49Vf+MhlnzX1vMCVlncGAxPPumpP33TqWY1FAGBKCv +5lrD2CNWEuDVUN/AXzl5qeFvOTkhOd99H6zZyidmvaD4sQOyrZ0m/+wRti4cE0GQ +O2Gfjf0yA2rWevc9XSGfyk1yBLlkVTD583zjTdFj1NguywH8UVhEQMJrHSPvmovc +Tsttkrwl+w/y6HV7c91HrhOC2hihMq7C0ip6WaH6vIBYWNpcs+g/8CpiTVVxEJ68 +5dQrDouKyR26SmWT1zLZw8tyb4Q/7YmzcHbxAgMBAAGjggE2MIIBMjAJBgNVHRME +AjAAMBEGCWCGSAGG+EIBAQQEAwIFoDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBH +ZW5lcmF0ZWQgQ2xpZW50IENlcnRpZmljYXRlMB0GA1UdDgQWBBSJ/yia067wCafe +kDCgk+e8PJTCUDAfBgNVHSMEGDAWgBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNV +HQ8BAf8EBAMCBeAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMEMDsGA1Ud +HwQ0MDIwMKAuoCyGKmh0dHA6Ly9sb2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUu +Y3JsLnBlbTAxBggrBgEFBQcBAQQlMCMwIQYIKwYBBQUHMAGGFWh0dHA6Ly9sb2Nh +bGhvc3Q6OTg3NzANBgkqhkiG9w0BAQsFAAOCAgEAN2XfYgbrjxC6OWh9UoMLQaDD +59JPxAUBxlRtWzTWqxY2jfT+OwJfDP4e+ef2G1YEG+qyt57ddlm/EwX9IvAvG0D4 +wd4tfItG88IJWKDM3wpT5KYrUsu+PlQTFmGmaWlORK/mRKlmfjbP5CIAcUedvCS9 +j9PkCrbbkklAmp0ULLSLUkYajmfFOkQ+VdGhQ6nAamTeyh2Z2S4dVjsKc8yBViMo +/V6HP56rOvUqiVTcvhZtH7QDptMSTzuJ+AsmreYjwIiTGzYS/i8QVAFuPfXJKEOB +jD5WhUaP/8Snbuft4MxssPAph8okcmxLfb55nw+soNc2oS1wWwKMe7igRelq8vtg +bu00QSEGiY1eq/vFgZh0+Wohy/YeYzhO4Jq40FFpKiVbkLzexpNH/Afj2QrHuZ7y +259uGGfv5tGA+TW6PsckCQknEb5V4V35ZZlbWVRKpuADeNPoDuoYPtc5eOomIkmw +rFz/gPZWSA+4pYEgXgqcaM8+KP0i53eTbWqwy5DVgXiuaTYWU4m1FTsIZ+/nGIqW +Dsgqd/D6jivf9Yvm+VFYTZsxIfq5sMdjxSuMBo0nZrzFDpqc6m6fVVoHv5R9Yliw +MbxgmFQ84CKLy7iNKGSGVN2SIr1obMQ0e/t3NiCHib3WKzmZFoNoFCtVzAgsxGmF +Q6rY83JdIPPW4LqZNcE= +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/index.txt b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/index.txt new file mode 100644 index 000000000..76a170dd3 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/index.txt @@ -0,0 +1,6 @@ +V 330419130816Z 1000 unknown /C=SE/ST=Stockholm/L=Stockholm/O=MyOrgName/OU=MyIntermediateCA/CN=localhost +V 330419130816Z 1001 unknown /C=SE/ST=Stockholm/L=Stockholm/O=MyOrgName/OU=MyIntermediateCA/CN=MyClient +R 330419130816Z 230112130816Z 1002 unknown /C=SE/ST=Stockholm/L=Stockholm/O=MyOrgName/OU=MyIntermediateCA/CN=client-revoked +V 330419130816Z 1003 unknown /C=SE/ST=Stockholm/L=Stockholm/O=MyOrgName/OU=MyIntermediateCA/CN=ocsp.server +V 330419130816Z 1004 unknown /C=SE/ST=Stockholm/L=Stockholm/O=MyOrgName/OU=MyIntermediateCA/CN=ocsp.client +V 330425123656Z 1005 unknown /C=SE/ST=Stockholm/L=Stockholm/O=MyOrgName/OU=MyIntermediateCA/CN=client-no-dist-points diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.key b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.key new file mode 100644 index 000000000..511cd5b0a --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC0Bu3THlP8vRwz +R84cnUdG5sDfVBqZaiTiT8KiUWmWXK5/bpoCfUlNopdnHhFcTA8MwVgkSiTufx/Q +PZNd0qrFzNnNnPcV3yBdZG/NtJQptVKMOYWf2rg48ZYjZPZDSQ4oAkYt0KCLP3VX +lsxbIBAtpYMUamrjdnjRW+v8l2O/PGACxwbAkVdjRThV8bT7Ly16Q/UP/BKmLEjL +iwEUXR1TVK8mhBeZYUoaXLs9E5SLk8tk4rlyVLSfqe2uGe364meibK5kFrFUAY+O +BfEyUR1s1gu75vXPncxF/Xqm6+Yjy/i+HLTFvS8+nj3GHyF53ughJbrEVH/BvsTN +bsKLxfW6VfvSKmO7QLvRtYmPYjbSYDTQyjgnDhinGQsegoNV2DhCkXjZ1DGs72Hq +ka641eVkNLznA+c3AUTU1wnXQi2u1NILlz/ZkK37pODUx9Qbe+yYnkyfHHw1x8LW +in+mhm5mT8JdV4vxrlzVD/8XqIO/ESshcAZo7diSsL1szKX5h+JhJBvZqIdmpb2v +mhM+iyCbj9eoOrNEOnO/d/zr1LaTXHsB8OFY59/Sw/aOlS6fOeBuJ/HVaVFkm1kv +rFw9WROvR4P7DDTMqvtPbn5G3sGPMaApv60Naag2QFTFrLLSRGmLu2/3m+0fkRjo +9WBAglf2KRGT3Q6LW0jUZCItnf2tTQIDAQABAoICAAVlH8Nv6TxtvmabBEY/QF+T +krwenR1z3N8bXM3Yer2S0XfoLJ1ee8/jy32/nO2TKfBL6wRLZIfxL1biQYRSR+Pd +m7lZtt3k7edelysm+jm1wV+KacK8n0C1nLY61FZ33gC88LV2xxjlMfMKBd3FPDbh ++ueluMZQSpablprfPpIAkTAEHuOud1v2OxX4RGAyrb44QyPTfguU0CmpZMLjd3mD +1CvnUX27OKlJliLib1UvfKztTnlqqG8QfJr3E/asykZH04IUXAQUd+TdsLi9TZBx +abCb30n1hKWkTwSplSAFgNLRsWkrnjrWKyvAyxQH5hT4OHyhu6JmwScW5qWhrRd3 +ld+pMaKQlOmtrTiRzSeFD2pOHFHvZ3N/1BhH5TGfnTIXKuEja3xdOArCHTBkh/9S +kEZegVIAjoFW+t3gfbz12JzNmDUUX+sWfadBBiwYepTUr2aZQehZM8+dzdSwQeh4 +XcAUC55YgaC2oFCfcc8rD5o+57nlR+7xAjZ/Z61SuUJHrKSRzB6w2PARiEIuYotK +E/CsQfL9tgjoc0aN0uVl8SH+GvKvRWM6LV711ep8w2XoPIAxId3ne/Ktw+wKCrqC +CJsHXIGOi8n0YZLZ6vz/6WrjmY1GdJc1aywQvr5eDFP5g0j3e+WzGBxoCKX8Gah5 +KpA4fcN44s2umsu7WcoBAoIBAQDZyGhtu9rbm3JMJrA9Eyq97niv6axwbhocE/bU +tPwdeWbPdprkK4aQ9UqJwHmVHkAUrGFRsY2iPJFLvdRwvixFYVAf/WLlAepd+HFz +Xit1oX5ouzbcjq2+13zUQpfjXFqfLqVYcu/sW7UFaD3yJEstkhI+ZM6Ci+kLWXN5 ++KOXASGzO8p7WBHFABRMH0bUjRnZy8xX3wdOhAKRFaCalxABodH9wz/cMunzrmEa +uHRsNWIIdWIVle4ZX4QTcsDgJSf5LeDaLtrpMu2AnFafQ2VCAb/jdKdighBsZG3H +Pu6e1fJzSKZEUtWSLMzBoB6R/oNDW9cPhcXWXlNc8QsZ7DAtAoIBAQDTnmUqf8Lo +lWPEQCrfkgQm2Gom/75uj5TnHsQYf2xk3vZNF5UwErD3Ixzh4F1E5ewA1Xvy5t3J +VCOLypiKDlfcZnsMPncdubGMrT575mkpZgsvR/w8u8pd4mFSdyCc/y5TeyfcNFQe +0Ho1NXMH6czutQs3oX+yfaTUr6Oa3brG1SAJQpG53nQI74pMWKHcivI/ytlA26Ki +zxIVzeAzJ/ToVc6MzbObkXjFxrnVlvjsLyGMJEfW2lmny4Gpx1xpc2j3YW8vehfx +DalWOJai1mtAo8ieo7CVw+kV2CqL7gJOJ2iNmCKT+IFk4LRtfJxd4wUJz6A/+vWp +o0LMvApAnIWhAoIBAER1S+Zaq9Rmi8pGSxYXxVLI+KULhkodQhXbbLa2YZ3+QIQs +m0noKLe+c3zTxSRLywb0nO7qKkR6V44AkRwTm6T/jwlPRFwKexqo8zi5vF2Qs0TG +vNsd+p3H7RRoDojIyi/JoO4pyyN4PHIDr51DLWKYzSVR2NyOkGYh6zvHHd1k3KwT +unWFXKiZesfm+QPtite8yXJByHE06/2hV8fgfoaU0Ia9boCQfJw+D4Yvv2EYcsWH +6JoydBMDxGe8pcaPx337nvfWzLeLa78G5e/QZq8WD7S3Qbqkefcopp2AOdAyHrGA +f8twYnQ9ouumopVv9OEiqHrXqTXWlsvbdYrjhM0CggEABOEHBhbSAJjJJxIvqt3r ++JVOxT1qP5RR445DCSmO7zhwx1A+4U/dAqWtmcuZeuguK8rAQ9Zs0KJ++08dezlf +bzZxqdOa3XWVkV/BLAwg6pJuuZVYTHIr9UQt6D/U4anEgKo7Pgl60wcNekKUN199 +mRdVfd/cWNoqvbia9gwcrU7moTAGuhlV5YrYTnBQswwFD9F2dtdZhZVunlAT1joa +nGy2CWsItBKDjVPKnxEPBisEA/4mJd786DB5+dcd21SM2/9EF/0hpi4hdFpzpqd4 +65GbI4U0og9VRWqpeHZxWSnxcCpMycqV+SRxJIEV/dgpGpPN5wu7NEEOXjgLqHez +YQKCAQBjwMVQUgn2KZK6Q9Lwe09ZpWTxGMh9mevU3eMA/6awajkE4UVgV8hSVvcG +i3Otn9UMnMhYu+HuU9O9W4zzncH0nRoiwjQr3X0MTT3Lc0rSJNPb/a6pcvysBuvB +wvhQ/dRXbCtmK9VE9ctPa9EO9f9SQRZF2NQsTOkyILdsgISm4zXSBhyT8KkQbiTe +0ToI7qMM73HqLHKOkjA+8jYkE5MTVQaaRXx2JlCeHEsIpH/2Nj1OsmUfn3paL6ZN +3loKhFfGy4onSOJOxoYaI3r6aykTFm7Qyg1xrG+8uFhK/qTOCB22I63LmSLZ1wlY +xBO4CmF79pAcAXvDoRB619Flx5/G +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.pem b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.pem new file mode 100644 index 000000000..467e4c209 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/ocsp-issuer.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF+zCCA+OgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwbzELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQK +DAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENBMREwDwYDVQQDDAhNeVJvb3RD +QTAeFw0yMzAxMTIxMzA4MTZaFw0zMzAxMDkxMzA4MTZaMGsxCzAJBgNVBAYTAlNF +MRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAoMCU15T3JnTmFtZTEZMBcGA1UE +CwwQTXlJbnRlcm1lZGlhdGVDQTEZMBcGA1UEAwwQTXlJbnRlcm1lZGlhdGVDQTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALQG7dMeU/y9HDNHzhydR0bm +wN9UGplqJOJPwqJRaZZcrn9umgJ9SU2il2ceEVxMDwzBWCRKJO5/H9A9k13SqsXM +2c2c9xXfIF1kb820lCm1Uow5hZ/auDjxliNk9kNJDigCRi3QoIs/dVeWzFsgEC2l +gxRqauN2eNFb6/yXY788YALHBsCRV2NFOFXxtPsvLXpD9Q/8EqYsSMuLARRdHVNU +ryaEF5lhShpcuz0TlIuTy2TiuXJUtJ+p7a4Z7friZ6JsrmQWsVQBj44F8TJRHWzW +C7vm9c+dzEX9eqbr5iPL+L4ctMW9Lz6ePcYfIXne6CElusRUf8G+xM1uwovF9bpV ++9IqY7tAu9G1iY9iNtJgNNDKOCcOGKcZCx6Cg1XYOEKReNnUMazvYeqRrrjV5WQ0 +vOcD5zcBRNTXCddCLa7U0guXP9mQrfuk4NTH1Bt77JieTJ8cfDXHwtaKf6aGbmZP +wl1Xi/GuXNUP/xeog78RKyFwBmjt2JKwvWzMpfmH4mEkG9moh2alva+aEz6LIJuP +16g6s0Q6c793/OvUtpNcewHw4Vjn39LD9o6VLp854G4n8dVpUWSbWS+sXD1ZE69H +g/sMNMyq+09ufkbewY8xoCm/rQ1pqDZAVMWsstJEaYu7b/eb7R+RGOj1YECCV/Yp +EZPdDotbSNRkIi2d/a1NAgMBAAGjgaQwgaEwHQYDVR0OBBYEFExwhjsVUom6tQ+S +qq6xMUETvnPzMB8GA1UdIwQYMBaAFD90kfU5pc5l48THu0Ayj9SNpHuhMBIGA1Ud +EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDsGA1UdHwQ0MDIwMKAuoCyG +Kmh0dHA6Ly9sb2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUuY3JsLnBlbTANBgkq +hkiG9w0BAQsFAAOCAgEAK6NgdWQYtPNKQNBGjsgtgqTRh+k30iqSO6Y3yE1KGABO +EuQdVqkC2qUIbCB0M0qoV0ab50KNLfU6cbshggW4LDpcMpoQpI05fukNh1jm3ZuZ +0xsB7vlmlsv00tpqmfIl/zykPDynHKOmFh/hJP/KetMy4+wDv4/+xP31UdEj5XvG +HvMtuqOS23A+H6WPU7ol7KzKBnU2zz/xekvPbUD3JqV+ynP5bgbIZHAndd0o9T8e +NFX23Us4cTenU2/ZlOq694bRzGaK+n3Ksz995Nbtzv5fbUgqmf7Mcq4iHGRVtV11 +MRyBrsXZp2vbF63c4hrf2Zd6SWRoaDKRhP2DMhajpH9zZASSTlfejg/ZRO2s+Clh +YrSTkeMAdnRt6i/q4QRcOTCfsX75RFM5v67njvTXsSaSTnAwaPi78tRtf+WSh0EP +VVPzy++BszBVlJ1VAf7soWZHCjZxZ8ZPqVTy5okoHwWQ09WmYe8GfulDh1oj0wbK +3FjN7bODWHJN+bFf5aQfK+tumYKoPG8RXL6QxpEzjFWjxhIMJHHMKfDWnAV1o1+7 +/1/aDzq7MzEYBbrgQR7oE5ZHtyqhCf9LUgw0Kr7/8QWuNAdeDCJzjXRROU0hJczp +dOyfRlLbHmLLmGOnROlx6LsGNQ17zuz6SPi7ei8/ylhykawDOAGkM1+xFakmQhM= +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf new file mode 100644 index 000000000..d26e12acf --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf @@ -0,0 +1,14 @@ +listeners.ssl.default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + keyfile = "{{ test_data_dir }}/server.key" + certfile = "{{ test_data_dir }}/server.pem" + cacertfile = "{{ test_data_dir }}/ca.pem" + ocsp { + enable_ocsp_stapling = true + issuer_pem = "{{ test_data_dir }}/ocsp-issuer.pem" + responder_url = "http://127.0.0.1:9877" + } + } +} diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.key b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.key new file mode 100644 index 000000000..d456ece72 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCnVPRWgP59GU15 +HddFwPZflFfcSkeuWU8tgKQhZcNoBli4lIfemuoV/hkGRVFexAiAw3/u5wvOaMaN +V8n9KxxgAUNLh5YaknpnNdhfQDyM0S5UJIbVeLzAQWxkBXpI3uBfW4WPSULRnVyR +psLEfl1qOklGOyuZfRbkkkkVwtJEmGEH0kz0fy6xenn3R3/mTeIbj+5TNqiBXWn1 +/qgTiNf2Ni7SE6Nk2lP4V8iofcBIrsp6KtEWdipGEJZeXCg/X0g/qVt15tF1l00M +uEWRHt1qGBELJJTcNzQvdqHAPz0AfQRjTtXyocw5+pFth8Q8a7gyjrjv5nhnpAKQ +msrt3vyNAgMBAAECggEABnWvIQ/Fw0qQxRYz00uJt1LguW5cqgxklBsdOvTUwFVO +Y4HIZP2R/9tZV/ahF4l10pK5g52DxSoiUB6Ne6qIY+RolqfbUZdKBmX7vmGadM02 +fqUSV3dbwghEiO/1Mo74FnZQB6IKZFEw26aWakN+k7VAUufB3SEJGzXSgHaO63ru +dFGSiYI8U+q+YnhUJjCnmI12fycNfy451TdUQtGZb6pNmm5HRUF6hpAV8Le9LojP +Ql9eacPpsrzU15X5ElCQZ/f9iNh1bplcISuhrULgKUKOvAVrBlEK67uRVy6g98xA +c/rgNLkbL/jZEsAc3/vHAyFgd3lABfwpBGLHej3QgQKBgQDFNYmfBNQr89HC5Zc+ +M6jXcAT/R+0GNczBTfC4iyNemwqsumSSRelNZ748UefKuS3F6Mvb2CBqE2LbB61G +hrnCffG2pARjZ491SefRwghhWWVGLP1p8KliLgOGBehA1REgJb+XULncjuHZuh4O +LVn3HVnWGxeBGg+yKa6Z4YQi3QKBgQDZN0O8ZcZY74lRJ0UjscD9mJ1yHlsssZag +njkX/f0GR/iVpfaIxQNC3gvWUy2LsU0He9sidcB0cfej0j/qZObQyFsCB0+utOgy ++hX7gokV2pes27WICbNWE2lJL4QZRJgvf82OaEy57kfDrm+eK1XaSZTZ10P82C9u +gAmMnontcQKBgGu29lhY9tqa7jOZ26Yp6Uri8JfO3XPK5u+edqEVvlfqL0Zw+IW8 +kdWpmIqx4f0kcA/tO4v03J+TvycLZmVjKQtGZ0PvCkaRRhY2K9yyMomZnmtaH4BB +5wKtR1do2pauyg/ZDnDDswD5OfsGYWw08TK8YVlEqu3lIjWZ9rguKVIxAoGAZYUk +zVqr10ks3pcCA2rCjkPT4lA5wKvHgI4ylPoKVfMxRY/pp4acvZXV5ne9o7pcDBFh +G7v5FPNnEFPlt4EtN4tMragJH9hBZgHoYEJkG6islweg0lHmVWaBIMlqbfzXO+v5 +gINSyNuLAvP2CvCqEXmubhnkFrpbgMOqsuQuBqECgYB3ss2PDhBF+5qoWgqymFof +1ovRPuQ9sPjWBn5IrCdoYITDnbBzBZERx7GLs6A/PUlWgST7jkb1PY/TxYSUfXzJ +SNd47q0mCQ+IUdqUbHgpK9b1ncwLMsnexpYZdHJWRLgnUhOx7OMjJc/4iLCAFCoN +3KJ7/V1keo7GBHOwnsFcCA== +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.pem b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.pem new file mode 100644 index 000000000..38cc63534 --- /dev/null +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/server.pem @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGCTCCA/GgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNloweDELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKdU9FaA/n0Z +TXkd10XA9l+UV9xKR65ZTy2ApCFlw2gGWLiUh96a6hX+GQZFUV7ECIDDf+7nC85o +xo1Xyf0rHGABQ0uHlhqSemc12F9APIzRLlQkhtV4vMBBbGQFekje4F9bhY9JQtGd +XJGmwsR+XWo6SUY7K5l9FuSSSRXC0kSYYQfSTPR/LrF6efdHf+ZN4huP7lM2qIFd +afX+qBOI1/Y2LtITo2TaU/hXyKh9wEiuynoq0RZ2KkYQll5cKD9fSD+pW3Xm0XWX +TQy4RZEe3WoYEQsklNw3NC92ocA/PQB9BGNO1fKhzDn6kW2HxDxruDKOuO/meGek +ApCayu3e/I0CAwEAAaOCAagwggGkMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD +AgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2Vy +dGlmaWNhdGUwHQYDVR0OBBYEFGy5LQPzIelruJl7mL0mtUXM57XhMIGaBgNVHSME +gZIwgY+AFExwhjsVUom6tQ+Sqq6xMUETvnPzoXOkcTBvMQswCQYDVQQGEwJTRTES +MBAGA1UECAwJU3RvY2tob2xtMRIwEAYDVQQHDAlTdG9ja2hvbG0xEjAQBgNVBAoM +CU15T3JnTmFtZTERMA8GA1UECwwITXlSb290Q0ExETAPBgNVBAMMCE15Um9vdENB +ggIQADAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwOwYDVR0f +BDQwMjAwoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4L2ludGVybWVkaWF0ZS5j +cmwucGVtMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDovL2xvY2Fs +aG9zdDo5ODc3MA0GCSqGSIb3DQEBCwUAA4ICAQCX3EQgiCVqLhnCNd0pmptxXPxo +l1KyZkpdrFa/NgSqRhkuZSAkszwBDDS/gzkHFKEUhmqs6/UZwN4+Rr3LzrHonBiN +aQ6GeNNXZ/3xAQfUCwjjGmz9Sgw6kaX19Gnk2CjI6xP7T+O5UmsMI9hHUepC9nWa +XX2a0hsO/KOVu5ZZckI16Ek/jxs2/HEN0epYdvjKFAaVmzZZ5PATNjrPQXvPmq2r +x++La+3bXZsrH8P2FhPpM5t/IxKKW/Tlpgz92c2jVSIHF5khSA/MFDC+dk80OFmm +v4ZTPIMuZ//Q+wo0f9P48rsL9D27qS7CA+8pn9wu+cfnBDSt7JD5Yipa1gHz71fy +YTa9qRxIAPpzW2v7TFZE8eSKFUY9ipCeM2BbdmCQGmq4+v36b5TZoyjH4k0UVWGo +Gclos2cic5Vxi8E6hb7b7yZpjEfn/5lbCiGMfAnI6aoOyrWg6keaRA33kaLUEZiK +OgFNbPkjiTV0ZQyLXf7uK9YFhpVzJ0dv0CFNse8rZb7A7PLn8VrV/ZFnJ9rPoawn +t7ZGxC0d5BRSEyEeEgsQdxuY4m8OkE18zwhCkt2Qs3uosOWlIrYmqSEa0i/sPSQP +jiwB4nEdBrf8ZygzuYjT5T9YRSwhVox4spS/Av8Ells5JnkuKAhCVv9gHxYwbj0c +CzyLJgE1z9Tq63m+gQ== +-----END CERTIFICATE----- diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index abab3f9b0..782f493fe 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -24,12 +24,15 @@ init_suite() -> init_suite([]). init_suite(Apps) -> - init_suite(Apps, fun set_special_configs/1). + init_suite(Apps, fun set_special_configs/1, #{}). -init_suite(Apps, SetConfigs) -> +init_suite(Apps, SetConfigs) when is_function(SetConfigs) -> + init_suite(Apps, SetConfigs, #{}). + +init_suite(Apps, SetConfigs, Opts) -> mria:start(), application:load(emqx_management), - emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], SetConfigs), + emqx_common_test_helpers:start_apps(Apps ++ [emqx_dashboard], SetConfigs, Opts), emqx_common_test_http:create_default_app(). end_suite() -> diff --git a/changes/ce/feat-10067.en.md b/changes/ce/feat-10067.en.md new file mode 100644 index 000000000..705e36137 --- /dev/null +++ b/changes/ce/feat-10067.en.md @@ -0,0 +1 @@ +Add support for OCSP stapling and CRL check for SSL MQTT listeners. diff --git a/changes/ce/feat-10067.zh.md b/changes/ce/feat-10067.zh.md new file mode 100644 index 000000000..d0efe4d5b --- /dev/null +++ b/changes/ce/feat-10067.zh.md @@ -0,0 +1 @@ +为SSL MQTT监听器增加对OCSP订书和CRL检查的支持。 diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 5975ebd1b..b027f92ec 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -12,6 +12,7 @@ CMD CN CONNACK CoAP +CRLs Cygwin DES DN @@ -41,6 +42,7 @@ Makefile MitM Multicast NIF +OCSP OTP PEM PINGREQ From 067747c2ded28bbb319753d6befcc2d13b4c698b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 10 Mar 2023 15:04:12 -0300 Subject: [PATCH 13/88] docs: improve descriptions Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx/i18n/emqx_schema_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 0690919bb..facccede2 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1755,8 +1755,8 @@ server_ssl_opts_schema_gc_after_handshake { server_ssl_opts_schema_enable_ocsp_stapling { desc { - en: "Whether to enable OCSP stapling for the listener. If set to true," - " requires defining the OCSP responder URL and issuer PEM path." + en: "Whether to enable Online Certificate Status Protocol (OCSP) stapling for the listener." + " If set to true, requires defining the OCSP responder URL and issuer PEM path." zh: "是否为监听器启用OCSP装订功能。 如果设置为 true，" "需要定义OCSP响应者URL和发行者PEM路径。" } From 57e38c850258cda0ad33171f3fc19415a5eaa043 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 09:24:55 -0300 Subject: [PATCH 14/88] refactor(ocsp): add reusable type for normalized binary URLs --- apps/emqx/rebar.config | 1 + apps/emqx/src/emqx_schema.erl | 26 +++++++++++------- apps/emqx/test/emqx_schema_tests.erl | 40 ++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index e782f714e..c1756b911 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,6 +30,7 @@ {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.4"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}}, + {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6412711a6..275a9592e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -43,6 +43,7 @@ -type cipher() :: map(). -type port_number() :: 1..65536. -type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}. +-type url() :: binary(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). @@ -56,6 +57,7 @@ -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). +-typerefl_from_string({url/0, emqx_schema, to_url}). -export([ validate_heap_size/1, @@ -81,7 +83,8 @@ to_bar_separated_list/1, to_ip_port/1, to_erl_cipher_suite/1, - to_comma_separated_atoms/1 + to_comma_separated_atoms/1, + to_url/1 ]). -export([ @@ -108,7 +111,8 @@ bar_separated_list/0, ip_port/0, cipher/0, - comma_separated_atoms/0 + comma_separated_atoms/0, + url/0 ]). -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). @@ -1306,16 +1310,9 @@ fields("ocsp") -> )}, {"responder_url", sc( - binary(), + url(), #{ required => false, - validator => fun ocsp_responder_url_validator/1, - converter => fun - (undefined, _Opts) -> - undefined; - (URL, _Opts) -> - uri_string:normalize(URL) - end, desc => ?DESC("server_ssl_opts_schema_ocsp_responder_url") } )}, @@ -2508,6 +2505,15 @@ to_comma_separated_binary(Str) -> to_comma_separated_atoms(Str) -> {ok, lists:map(fun to_atom/1, string:tokens(Str, ", "))}. +to_url(Str) -> + case emqx_http_lib:uri_parse(Str) of + {ok, URIMap} -> + URIString = emqx_http_lib:normalize(URIMap), + {ok, iolist_to_binary(URIString)}; + Error -> + Error + end. + to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index a0d264662..5176f4fad 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -473,3 +473,43 @@ password_converter_test() -> ?assertEqual(<<"123">>, emqx_schema:password_converter(<<"123">>, #{})), ?assertThrow("must_quote", emqx_schema:password_converter(foobar, #{})), ok. + +url_type_test_() -> + [ + ?_assertEqual( + {ok, <<"http://some.server/">>}, + typerefl:from_string(emqx_schema:url(), <<"http://some.server/">>) + ), + ?_assertEqual( + {ok, <<"http://192.168.0.1/">>}, + typerefl:from_string(emqx_schema:url(), <<"http://192.168.0.1">>) + ), + ?_assertEqual( + {ok, <<"http://some.server/">>}, + typerefl:from_string(emqx_schema:url(), "http://some.server/") + ), + ?_assertEqual( + {ok, <<"http://some.server/">>}, + typerefl:from_string(emqx_schema:url(), <<"http://some.server">>) + ), + ?_assertEqual( + {ok, <<"http://some.server:9090/">>}, + typerefl:from_string(emqx_schema:url(), <<"http://some.server:9090">>) + ), + ?_assertEqual( + {ok, <<"https://some.server:9090/">>}, + typerefl:from_string(emqx_schema:url(), <<"https://some.server:9090">>) + ), + ?_assertEqual( + {ok, <<"https://some.server:9090/path?q=uery">>}, + typerefl:from_string(emqx_schema:url(), <<"https://some.server:9090/path?q=uery">>) + ), + ?_assertEqual( + {error, {unsupported_scheme, <<"postgres">>}}, + typerefl:from_string(emqx_schema:url(), <<"postgres://some.server:9090">>) + ), + ?_assertEqual( + {error, empty_host_not_allowed}, + typerefl:from_string(emqx_schema:url(), <<"">>) + ) + ]. From 158f054187ecf1c1f3ce71ae9caca44098c0dbe1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 14:07:41 -0300 Subject: [PATCH 15/88] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index facccede2..ce76aff3c 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1757,34 +1757,34 @@ server_ssl_opts_schema_enable_ocsp_stapling { desc { en: "Whether to enable Online Certificate Status Protocol (OCSP) stapling for the listener." " If set to true, requires defining the OCSP responder URL and issuer PEM path." - zh: "是否为监听器启用OCSP装订功能。 如果设置为 true，" - "需要定义OCSP响应者URL和发行者PEM路径。" + zh: "是否为监听器启用 OCSP Stapling 功能。 如果设置为 true，" + "需要定义 OCSP Responder 的 URL 和证书签发者的 PEM 文件路径。" } label: { en: "Enable OCSP Stapling" - zh: "启用OCSP订书机" + zh: "启用 OCSP Stapling" } } server_ssl_opts_schema_ocsp_responder_url { desc { en: "URL for the OCSP responder to check the server certificate against." - zh: "用于检查服务器证书的OCSP响应器的URL。" + zh: "用于检查服务器证书的 OCSP Responder 的 URL。" } label: { en: "OCSP Responder URL" - zh: "OCSP响应者URL" + zh: "OCSP Responder 的 URL" } } server_ssl_opts_schema_ocsp_issuer_pem { desc { en: "PEM-encoded certificate of the OCSP issuer for the server certificate." - zh: "服务器证书的OCSP签发者的PEM编码证书。" + zh: "服务器证书的 OCSP 签发者的 PEM 编码证书。" } label: { en: "OCSP Issuer Certificate" - zh: "OCSP发行人证书" + zh: "OCSP 签发者证书" } } From 63ef2f9b79e21eeec2e042fe31e67b8134c533a2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 3 Mar 2023 11:54:22 -0300 Subject: [PATCH 16/88] feat: save uploaded OCSP issuer pem like other ssl certs --- apps/emqx/src/emqx_tls_lib.erl | 88 ++++++++++++------- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 22 ++++- apps/emqx/test/emqx_tls_lib_tests.erl | 55 +++++++++--- .../src/emqx_dashboard_listener.erl | 2 +- 4 files changed, 118 insertions(+), 49 deletions(-) diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index eb6091f29..eb8234547 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -47,8 +47,18 @@ -define(IS_TRUE(Val), ((Val =:= true) orelse (Val =:= <<"true">>))). -define(IS_FALSE(Val), ((Val =:= false) orelse (Val =:= <<"false">>))). --define(SSL_FILE_OPT_NAMES, [<<"keyfile">>, <<"certfile">>, <<"cacertfile">>]). --define(SSL_FILE_OPT_NAMES_A, [keyfile, certfile, cacertfile]). +-define(SSL_FILE_OPT_NAMES, [ + [<<"keyfile">>], + [<<"certfile">>], + [<<"cacertfile">>], + [<<"ocsp">>, <<"issuer_pem">>] +]). +-define(SSL_FILE_OPT_NAMES_A, [ + [keyfile], + [certfile], + [cacertfile], + [ocsp, issuer_pem] +]). %% non-empty string -define(IS_STRING(L), (is_list(L) andalso L =/= [] andalso is_integer(hd(L)))). @@ -298,20 +308,20 @@ ensure_ssl_files(Dir, SSL, Opts) -> RequiredKeys = maps:get(required_keys, Opts, []), case ensure_ssl_file_key(SSL, RequiredKeys) of ok -> - Keys = ?SSL_FILE_OPT_NAMES ++ ?SSL_FILE_OPT_NAMES_A, - ensure_ssl_files(Dir, SSL, Keys, Opts); + KeyPaths = ?SSL_FILE_OPT_NAMES ++ ?SSL_FILE_OPT_NAMES_A, + ensure_ssl_files(Dir, SSL, KeyPaths, Opts); {error, _} = Error -> Error end. ensure_ssl_files(_Dir, SSL, [], _Opts) -> {ok, SSL}; -ensure_ssl_files(Dir, SSL, [Key | Keys], Opts) -> - case ensure_ssl_file(Dir, Key, SSL, maps:get(Key, SSL, undefined), Opts) of +ensure_ssl_files(Dir, SSL, [KeyPath | KeyPaths], Opts) -> + case ensure_ssl_file(Dir, KeyPath, SSL, emqx_map_lib:deep_get(KeyPath, SSL, undefined), Opts) of {ok, NewSSL} -> - ensure_ssl_files(Dir, NewSSL, Keys, Opts); + ensure_ssl_files(Dir, NewSSL, KeyPaths, Opts); {error, Reason} -> - {error, Reason#{which_options => [Key]}} + {error, Reason#{which_options => [KeyPath]}} end. %% @doc Compare old and new config, delete the ones in old but not in new. @@ -321,11 +331,11 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> {ok, NewOpts} = ensure_ssl_files(Dir, NewOpts0, #{dry_run => DryRun}), {ok, OldOpts} = ensure_ssl_files(Dir, OldOpts0, #{dry_run => DryRun}), Get = fun - (_K, undefined) -> undefined; - (K, Opts) -> maps:get(K, Opts, undefined) + (_KP, undefined) -> undefined; + (KP, Opts) -> emqx_map_lib:deep_get(KP, Opts, undefined) end, lists:foreach( - fun(Key) -> delete_old_file(Get(Key, NewOpts), Get(Key, OldOpts)) end, + fun(KeyPath) -> delete_old_file(Get(KeyPath, NewOpts), Get(KeyPath, OldOpts)) end, ?SSL_FILE_OPT_NAMES ++ ?SSL_FILE_OPT_NAMES_A ), %% try to delete the dir if it is empty @@ -346,29 +356,33 @@ delete_old_file(_New, Old) -> ?SLOG(error, #{msg => "failed_to_delete_ssl_file", file_path => Old, reason => Reason}) end. -ensure_ssl_file(_Dir, _Key, SSL, undefined, _Opts) -> +ensure_ssl_file(_Dir, _KeyPath, SSL, undefined, _Opts) -> {ok, SSL}; -ensure_ssl_file(Dir, Key, SSL, MaybePem, Opts) -> +ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, Opts) -> case is_valid_string(MaybePem) of true -> DryRun = maps:get(dry_run, Opts, false), - do_ensure_ssl_file(Dir, Key, SSL, MaybePem, DryRun); + do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun); false -> {error, #{reason => invalid_file_path_or_pem_string}} end. -do_ensure_ssl_file(Dir, Key, SSL, MaybePem, DryRun) -> +do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) -> case is_pem(MaybePem) of true -> - case save_pem_file(Dir, Key, MaybePem, DryRun) of - {ok, Path} -> {ok, SSL#{Key => Path}}; - {error, Reason} -> {error, Reason} + case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of + {ok, Path} -> + NewSSL = emqx_map_lib:deep_put(KeyPath, SSL, Path), + {ok, NewSSL}; + {error, Reason} -> + {error, Reason} end; false -> case is_valid_pem_file(MaybePem) of true -> {ok, SSL}; - {error, enoent} when DryRun -> {ok, SSL}; + {error, enoent} when DryRun -> + {ok, SSL}; {error, Reason} -> {error, #{ pem_check => invalid_pem, @@ -398,8 +412,8 @@ is_pem(MaybePem) -> %% To make it simple, the file is always overwritten. %% Also a potentially half-written PEM file (e.g. due to power outage) %% can be corrected with an overwrite. -save_pem_file(Dir, Key, Pem, DryRun) -> - Path = pem_file_name(Dir, Key, Pem), +save_pem_file(Dir, KeyPath, Pem, DryRun) -> + Path = pem_file_name(Dir, KeyPath, Pem), case filelib:ensure_dir(Path) of ok when DryRun -> {ok, Path}; @@ -422,11 +436,14 @@ is_generated_file(Filename) -> _ -> false end. -pem_file_name(Dir, Key, Pem) -> +pem_file_name(Dir, KeyPath, Pem) -> <> = crypto:hash(md5, Pem), Suffix = hex_str(CK), - FileName = binary:replace(ensure_bin(Key), <<"file">>, <<"-", Suffix/binary>>), - filename:join([pem_dir(Dir), FileName]). + Segments = lists:map(fun ensure_bin/1, KeyPath), + Filename0 = iolist_to_binary(lists:join(<<"_">>, Segments)), + Filename1 = binary:replace(Filename0, <<"file">>, <<>>), + Filename = <>, + filename:join([pem_dir(Dir), Filename]). pem_dir(Dir) -> filename:join([emqx:mutable_certs_dir(), Dir]). @@ -465,9 +482,9 @@ is_valid_pem_file(Path) -> %% so they are forced to upload a cert file, or use an existing file path. -spec drop_invalid_certs(map()) -> map(). drop_invalid_certs(#{enable := False} = SSL) when ?IS_FALSE(False) -> - maps:without(?SSL_FILE_OPT_NAMES_A, SSL); + lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_NAMES_A); drop_invalid_certs(#{<<"enable">> := False} = SSL) when ?IS_FALSE(False) -> - maps:without(?SSL_FILE_OPT_NAMES, SSL); + lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_NAMES); drop_invalid_certs(#{enable := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs(?SSL_FILE_OPT_NAMES_A, SSL); drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> @@ -475,14 +492,16 @@ drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs([], SSL) -> SSL; -do_drop_invalid_certs([Key | Keys], SSL) -> - case maps:get(Key, SSL, undefined) of +do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> + case emqx_map_lib:deep_get(KeyPath, SSL, undefined) of undefined -> - do_drop_invalid_certs(Keys, SSL); + do_drop_invalid_certs(KeyPaths, SSL); PemOrPath -> case is_pem(PemOrPath) orelse is_valid_pem_file(PemOrPath) of - true -> do_drop_invalid_certs(Keys, SSL); - {error, _} -> do_drop_invalid_certs(Keys, maps:without([Key], SSL)) + true -> + do_drop_invalid_certs(KeyPaths, SSL); + {error, _} -> + do_drop_invalid_certs(KeyPaths, emqx_map_lib:deep_remove(KeyPath, SSL)) end end. @@ -565,9 +584,10 @@ ensure_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). ensure_ssl_file_key(_SSL, []) -> ok; -ensure_ssl_file_key(SSL, RequiredKeys) -> - Filter = fun(Key) -> not maps:is_key(Key, SSL) end, - case lists:filter(Filter, RequiredKeys) of +ensure_ssl_file_key(SSL, RequiredKeyPaths) -> + NotFoundRef = make_ref(), + Filter = fun(KeyPath) -> NotFoundRef =:= emqx_map_lib:deep_get(KeyPath, SSL, NotFoundRef) end, + case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} end. diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 1f5e34548..3b1c2adaf 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -673,7 +673,8 @@ do_t_update_listener(Config) -> Keyfile = filename:join([DataDir, "server.key"]), Certfile = filename:join([DataDir, "server.pem"]), Cacertfile = filename:join([DataDir, "ca.pem"]), - IssuerPem = filename:join([DataDir, "ocsp-issuer.pem"]), + IssuerPemPath = filename:join([DataDir, "ocsp-issuer.pem"]), + {ok, IssuerPem} = file:read_file(IssuerPemPath), %% no ocsp at first ListenerId = "ssl:default", @@ -701,6 +702,9 @@ do_t_update_listener(Config) -> <<"ocsp">> => #{ <<"enable_ocsp_stapling">> => true, + %% we use the file contents to check that + %% the API converts that to an internally + %% managed file <<"issuer_pem">> => IssuerPem, <<"responder_url">> => <<"http://localhost:9877">> } @@ -722,6 +726,22 @@ do_t_update_listener(Config) -> }, ListenerData2 ), + %% issuer pem should have been uploaded and saved to a new + %% location + ?assertNotEqual( + IssuerPemPath, + emqx_map_lib:deep_get( + [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], + ListenerData2 + ) + ), + ?assertNotEqual( + IssuerPem, + emqx_map_lib:deep_get( + [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], + ListenerData2 + ) + ), assert_http_get(1, 5_000), ok. diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 5a81daf6a..5510e4027 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -117,7 +117,7 @@ ssl_files_failure_test_() -> %% empty string ?assertMatch( {error, #{ - reason := invalid_file_path_or_pem_string, which_options := [<<"keyfile">>] + reason := invalid_file_path_or_pem_string, which_options := [[<<"keyfile">>]] }}, emqx_tls_lib:ensure_ssl_files("/tmp", #{ <<"keyfile">> => <<>>, @@ -128,7 +128,7 @@ ssl_files_failure_test_() -> %% not valid unicode ?assertMatch( {error, #{ - reason := invalid_file_path_or_pem_string, which_options := [<<"keyfile">>] + reason := invalid_file_path_or_pem_string, which_options := [[<<"keyfile">>]] }}, emqx_tls_lib:ensure_ssl_files("/tmp", #{ <<"keyfile">> => <<255, 255>>, @@ -136,6 +136,18 @@ ssl_files_failure_test_() -> <<"cacertfile">> => bin(test_key()) }) ), + ?assertMatch( + {error, #{ + reason := invalid_file_path_or_pem_string, + which_options := [[<<"ocsp">>, <<"issuer_pem">>]] + }}, + emqx_tls_lib:ensure_ssl_files("/tmp", #{ + <<"keyfile">> => bin(test_key()), + <<"certfile">> => bin(test_key()), + <<"cacertfile">> => bin(test_key()), + <<"ocsp">> => #{<<"issuer_pem">> => <<255, 255>>} + }) + ), %% not printable ?assertMatch( {error, #{reason := invalid_file_path_or_pem_string}}, @@ -155,7 +167,8 @@ ssl_files_failure_test_() -> #{ <<"cacertfile">> => bin(TmpFile), <<"keyfile">> => bin(TmpFile), - <<"certfile">> => bin(TmpFile) + <<"certfile">> => bin(TmpFile), + <<"ocsp">> => #{<<"issuer_pem">> => bin(TmpFile)} } ) ) @@ -170,22 +183,29 @@ ssl_files_save_delete_test() -> SSL0 = #{ <<"keyfile">> => Key, <<"certfile">> => Key, - <<"cacertfile">> => Key + <<"cacertfile">> => Key, + <<"ocsp">> => #{<<"issuer_pem">> => Key} }, Dir = filename:join(["/tmp", "ssl-test-dir"]), {ok, SSL} = emqx_tls_lib:ensure_ssl_files(Dir, SSL0), - File = maps:get(<<"keyfile">>, SSL), - ?assertMatch(<<"/tmp/ssl-test-dir/key-", _:16/binary>>, File), - ?assertEqual({ok, bin(test_key())}, file:read_file(File)), + FileKey = maps:get(<<"keyfile">>, SSL), + ?assertMatch(<<"/tmp/ssl-test-dir/key-", _:16/binary>>, FileKey), + ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), + FileIssuerPem = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), + ?assertMatch(<<"/tmp/ssl-test-dir/ocsp_issuer_pem-", _:16/binary>>, FileIssuerPem), + ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% no old file to delete ok = emqx_tls_lib:delete_ssl_files(Dir, SSL, undefined), - ?assertEqual({ok, bin(test_key())}, file:read_file(File)), + ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), + ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% old and new identical, no delete ok = emqx_tls_lib:delete_ssl_files(Dir, SSL, SSL), - ?assertEqual({ok, bin(test_key())}, file:read_file(File)), + ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), + ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% new is gone, delete old ok = emqx_tls_lib:delete_ssl_files(Dir, undefined, SSL), - ?assertEqual({error, enoent}, file:read_file(File)), + ?assertEqual({error, enoent}, file:read_file(FileKey)), + ?assertEqual({error, enoent}, file:read_file(FileIssuerPem)), %% test idempotence ok = emqx_tls_lib:delete_ssl_files(Dir, undefined, SSL), ok. @@ -198,7 +218,8 @@ ssl_files_handle_non_generated_file_test() -> SSL0 = #{ <<"keyfile">> => TmpKeyFile, <<"certfile">> => TmpKeyFile, - <<"cacertfile">> => TmpKeyFile + <<"cacertfile">> => TmpKeyFile, + <<"ocsp">> => #{<<"issuer_pem">> => TmpKeyFile} }, Dir = filename:join(["/tmp", "ssl-test-dir-00"]), {ok, SSL2} = emqx_tls_lib:ensure_ssl_files(Dir, SSL0), @@ -216,24 +237,32 @@ ssl_file_replace_test() -> SSL0 = #{ <<"keyfile">> => Key1, <<"certfile">> => Key1, - <<"cacertfile">> => Key1 + <<"cacertfile">> => Key1, + <<"ocsp">> => #{<<"issuer_pem">> => Key1} }, SSL1 = #{ <<"keyfile">> => Key2, <<"certfile">> => Key2, - <<"cacertfile">> => Key2 + <<"cacertfile">> => Key2, + <<"ocsp">> => #{<<"issuer_pem">> => Key2} }, Dir = filename:join(["/tmp", "ssl-test-dir2"]), {ok, SSL2} = emqx_tls_lib:ensure_ssl_files(Dir, SSL0), {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1), File1 = maps:get(<<"keyfile">>, SSL2), File2 = maps:get(<<"keyfile">>, SSL3), + IssuerPem1 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), + IssuerPem2 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), ?assert(filelib:is_regular(File1)), ?assert(filelib:is_regular(File2)), + ?assert(filelib:is_regular(IssuerPem1)), + ?assert(filelib:is_regular(IssuerPem2)), %% delete old file (File1, in SSL2) ok = emqx_tls_lib:delete_ssl_files(Dir, SSL3, SSL2), ?assertNot(filelib:is_regular(File1)), ?assert(filelib:is_regular(File2)), + ?assertNot(filelib:is_regular(IssuerPem1)), + ?assert(filelib:is_regular(IssuerPem2)), ok. bin(X) -> iolist_to_binary(X). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index 112b3ad58..eac4f845f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -163,7 +163,7 @@ diff_listeners(Type, Stop, Start) -> {#{Type => Stop}, #{Type => Start}}. ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"enable">> := true}}} = Conf) -> Https = emqx_map_lib:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), - Opts = #{required_keys => [<<"keyfile">>, <<"certfile">>, <<"cacertfile">>]}, + Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]}, case emqx_tls_lib:ensure_ssl_files(?DIR, Https, Opts) of {ok, undefined} -> {error, <<"ssl_cert_not_found">>}; From f6707d1dd09baa19d5b3439342c984c8ee28707c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 7 Mar 2023 10:17:33 -0300 Subject: [PATCH 17/88] test: fix how ocsp client is run in tests For some yet unknown reason the old test version using `open_port` does not work in OTP 25, but works fine in OTP 24. There are no messages at all received from The openssl client port program in OTP 25. --- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 115 ++++++++++------------- 1 file changed, 49 insertions(+), 66 deletions(-) diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 3b1c2adaf..890d2ecaa 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -263,7 +263,7 @@ assert_http_get(N, Timeout) when N > 0 -> end, assert_http_get(N - 1, Timeout). -spawn_openssl_client(TLSVsn, RequestStatus, Config) -> +openssl_client_command(TLSVsn, RequestStatus, Config) -> DataDir = ?config(data_dir, Config), ClientCert = filename:join([DataDir, "client.pem"]), ClientKey = filename:join([DataDir, "client.key"]), @@ -274,25 +274,38 @@ spawn_openssl_client(TLSVsn, RequestStatus, Config) -> true -> ["-status"]; false -> [] end, + [ + Openssl, + "s_client", + "-connect", + "localhost:8883", + %% needed to trigger `sni_fun' + "-servername", + "localhost", + TLSVsn, + "-CAfile", + Cacert, + "-cert", + ClientCert, + "-key", + ClientKey + ] ++ StatusOpt. + +run_openssl_client(TLSVsn, RequestStatus, Config) -> + Command0 = openssl_client_command(TLSVsn, RequestStatus, Config), + Command = lists:flatten(lists:join(" ", Command0)), + os:cmd(Command). + +%% fixme: for some reason, the port program doesn't return any output +%% when running in OTP 25 using `open_port`, but the `os:cmd` version +%% works fine. +%% the `open_port' version works fine in OTP 24 for some reason. +spawn_openssl_client(TLSVsn, RequestStatus, Config) -> + [Openssl | Args] = openssl_client_command(TLSVsn, RequestStatus, Config), open_port( {spawn_executable, Openssl}, [ - {args, - [ - "s_client", - "-connect", - "localhost:8883", - %% needed to trigger `sni_fun' - "-servername", - "localhost", - TLSVsn, - "-CAfile", - Cacert, - "-cert", - ClientCert, - "-key", - ClientKey - ] ++ StatusOpt}, + {args, Args}, binary, stderr_to_stdout ] @@ -331,56 +344,26 @@ kill_pid(OSPid) -> os:cmd("kill -9 " ++ integer_to_list(OSPid)). test_ocsp_connection(TLSVsn, WithRequestStatus = true, Config) -> - ClientPort = spawn_openssl_client(TLSVsn, WithRequestStatus, Config), - {os_pid, ClientOSPid} = erlang:port_info(ClientPort, os_pid), - try - timer:sleep(timer:seconds(1)), - {messages, Messages} = process_info(self(), messages), - OCSPOutput0 = [ - Output - || {_Port, {data, Output}} <- Messages, - re:run(Output, "OCSP response:") =/= nomatch - ], - ?assertMatch( - [_], - OCSPOutput0, - #{all_messages => Messages} - ), - [OCSPOutput] = OCSPOutput0, - ?assertMatch( - {match, _}, - re:run(OCSPOutput, "OCSP Response Status: successful"), - #{all_messages => Messages} - ), - ?assertMatch( - {match, _}, - re:run(OCSPOutput, "Cert Status: good"), - #{all_messages => Messages} - ), - ok - after - catch kill_pid(ClientOSPid) - end; + OCSPOutput = run_openssl_client(TLSVsn, WithRequestStatus, Config), + ?assertMatch( + {match, _}, + re:run(OCSPOutput, "OCSP Response Status: successful"), + #{mailbox => process_info(self(), messages)} + ), + ?assertMatch( + {match, _}, + re:run(OCSPOutput, "Cert Status: good"), + #{mailbox => process_info(self(), messages)} + ), + ok; test_ocsp_connection(TLSVsn, WithRequestStatus = false, Config) -> - ClientPort = spawn_openssl_client(TLSVsn, WithRequestStatus, Config), - {os_pid, ClientOSPid} = erlang:port_info(ClientPort, os_pid), - try - timer:sleep(timer:seconds(1)), - {messages, Messages} = process_info(self(), messages), - OCSPOutput = [ - Output - || {_Port, {data, Output}} <- Messages, - re:run(Output, "OCSP response:") =/= nomatch - ], - ?assertEqual( - [], - OCSPOutput, - #{all_messages => Messages} - ), - ok - after - catch kill_pid(ClientOSPid) - end. + OCSPOutput = run_openssl_client(TLSVsn, WithRequestStatus, Config), + ?assertMatch( + nomatch, + re:run(OCSPOutput, "Cert Status: good", [{capture, none}]), + #{mailbox => process_info(self(), messages)} + ), + ok. ensure_port_open(Port) -> do_ensure_port_open(Port, 10). From 04378f242c2cd9c1d31981ba4cbe24a976ef17f7 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 14 Mar 2023 09:22:41 -0300 Subject: [PATCH 18/88] feat(ocsp_cache): give ocsp cache table an heir --- apps/emqx/src/emqx_ocsp_cache.erl | 5 +++-- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 2fe3aa5d5..25c6200ae 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -149,9 +149,10 @@ inject_sni_fun(ListenerID, Conf0) -> init(_Args) -> logger:set_process_metadata(#{domain => [emqx, ocsp, cache]}), - _ = ets:new(?CACHE_TAB, [ + emqx_tables:new(?CACHE_TAB, [ named_table, - protected, + public, + {heir, whereis(emqx_kernel_sup), none}, {read_concurrency, true} ]), ?tp(ocsp_cache_init, #{}), diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 890d2ecaa..e0c29e440 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -147,6 +147,7 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), + _Heir = spawn_dummy_heir(), {ok, CachePid} = emqx_ocsp_cache:start_link(), [ {cache_pid, CachePid} @@ -164,6 +165,7 @@ init_per_testcase(_TestCase, Config) -> {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} end ), + _Heir = spawn_dummy_heir(), {ok, CachePid} = emqx_ocsp_cache:start_link(), DataDir = ?config(data_dir, Config), Type = ssl, @@ -225,6 +227,17 @@ end_per_testcase(_TestCase, Config) -> %% Helper functions %%-------------------------------------------------------------------- +%% The real cache makes `emqx_kernel_sup' the heir to its ETS table. +%% In some tests, we don't start the full supervision tree, so we need +%% this dummy process. +spawn_dummy_heir() -> + spawn_link(fun() -> + true = register(emqx_kernel_sup, self()), + receive + stop -> ok + end + end). + does_module_exist(Mod) -> case erlang:module_loaded(Mod) of true -> @@ -508,11 +521,13 @@ t_request_ocsp_response_restart_cache(Config) -> ok end, fun(Trace) -> + %% Only one fetch because the cache table was preserved by + %% its heir ("emqx_kernel_sup"). ?assertMatch( - [_, _], + [_], ?of_kind(ocsp_http_fetch_and_cache, Trace) ), - assert_http_get(2), + assert_http_get(1), ok end ). From 13a35cc0455ca3f243ed98c3a11c6075bd385306 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 14 Mar 2023 12:17:08 +0100 Subject: [PATCH 19/88] fix: schema default value so that it shows up correctly in UI This complements PR https://github.com/emqx/emqx/pull/10124. The default values for duration_ms() fields needs to be formatted as a binary string with unit to show up correctly in the dashboard UI. --- apps/emqx_connector/src/emqx_connector_mongo.erl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 8804ebaf2..ead3e2e49 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -106,7 +106,14 @@ fields(topology) -> {socket_timeout_ms, duration("socket_timeout")}, {server_selection_timeout_ms, duration("server_selection_timeout")}, {wait_queue_timeout_ms, duration("wait_queue_timeout")}, - {heartbeat_frequency_ms, fun heartbeat_frequency_ms/1}, + {heartbeat_frequency_ms, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"200s">>, + desc => ?DESC("heartbeat_period") + } + )}, {min_heartbeat_frequency_ms, duration("min_heartbeat_period")} ]. @@ -407,12 +414,6 @@ duration(Desc) -> desc => ?DESC(Desc) }. -heartbeat_frequency_ms(type) -> emqx_schema:duration_ms(); -heartbeat_frequency_ms(desc) -> ?DESC("heartbeat_period"); -heartbeat_frequency_ms(default) -> 200000; -heartbeat_frequency_ms(validator) -> [?MIN(1)]; -heartbeat_frequency_ms(_) -> undefined. - max_overflow(type) -> non_neg_integer(); max_overflow(desc) -> ?DESC("max_overflow"); max_overflow(default) -> 0; From aac41ba1281a43bbbd29e7548d5c937268d44fb0 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 11:35:13 +0100 Subject: [PATCH 20/88] fix: emqx ctl crashed when $HOME/.erlang.cookie is not available fixes #10142 --- bin/emqx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index 14f94f359..741aa3718 100755 --- a/bin/emqx +++ b/bin/emqx @@ -441,9 +441,8 @@ call_nodetool() { # Control a node relx_nodetool() { command="$1"; shift - ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARGS" \ - call_nodetool "$NAME_TYPE" "$NAME" \ - -setcookie "$COOKIE" "$command" "$@" + ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARGS -setcookie $COOKIE" \ + call_nodetool "$NAME_TYPE" "$NAME" "$command" "$@" } call_hocon() { From fc2ddc634955192b053e463305e6d31fae3f961b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 15 Mar 2023 20:36:09 +0800 Subject: [PATCH 21/88] chore: add changelog for configuration information on Dashboard is garbled --- changes/ce/fix-10130.en.md | 4 ++++ changes/ce/fix-10130.zh.md | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 changes/ce/fix-10130.en.md create mode 100644 changes/ce/fix-10130.zh.md diff --git a/changes/ce/fix-10130.en.md b/changes/ce/fix-10130.en.md new file mode 100644 index 000000000..d829e2c72 --- /dev/null +++ b/changes/ce/fix-10130.en.md @@ -0,0 +1,4 @@ +Fix the EMQX node started by environment variable configuration cannot get the correct configuration information through HTTP API. +For example: `EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` +and the Statsd configuration information on Dashboard is garbled (not '127.0.0.1:8124'). +Related PR: [HOCON:234](https://github.com/emqx/hocon/pull/234). diff --git a/changes/ce/fix-10130.zh.md b/changes/ce/fix-10130.zh.md new file mode 100644 index 000000000..19c092fdf --- /dev/null +++ b/changes/ce/fix-10130.zh.md @@ -0,0 +1,3 @@ +修复通过环境变量配置启动的 EMQX 节点无法通过HTTP API获取到正确的配置信息。 +比如：`EMQX_STATSD__SERVER='127.0.0.1:8124' ./bin/emqx start` 后通过 Dashboard看到的 Statsd 配置信息是乱码。 +相关 PR: [HOCON:234](https://github.com/emqx/hocon/pull/234). From d43ee0be60a23a1a3960a38aed220cc8dbea98d3 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 11:38:21 +0100 Subject: [PATCH 22/88] chore: add changelog --- changes/ce/fix-10144.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10144.en.md diff --git a/changes/ce/fix-10144.en.md b/changes/ce/fix-10144.en.md new file mode 100644 index 000000000..d5a84b24c --- /dev/null +++ b/changes/ce/fix-10144.en.md @@ -0,0 +1 @@ +Add -setcookie emulator flag when invoking emqx ctl to prevent problems with emqx cli when home directory is read only. Fixes [#10142](https://github.com/emqx/emqx/issues/10142) From 60677bc4009732e5409e0d7c263e99f34c0ff574 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 15 Mar 2023 20:36:40 +0800 Subject: [PATCH 23/88] chore: remove unuse code --- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 1 - apps/emqx_plugins/src/emqx_plugins_sup.erl | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index de93a1071..2e6aac849 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -49,7 +49,6 @@ <<"authorization">>, <<"authentication">>, <<"rpc">>, - <<"db">>, <<"connectors">>, <<"slow_subs">>, <<"psk_authentication">>, diff --git a/apps/emqx_plugins/src/emqx_plugins_sup.erl b/apps/emqx_plugins/src/emqx_plugins_sup.erl index 31427aaf6..f22daa9b8 100644 --- a/apps/emqx_plugins/src/emqx_plugins_sup.erl +++ b/apps/emqx_plugins/src/emqx_plugins_sup.erl @@ -26,18 +26,6 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - %% TODO: Add monitor plugins change. - Monitor = emqx_plugins_monitor, - _Children = [ - #{ - id => Monitor, - start => {Monitor, start_link, []}, - restart => permanent, - shutdown => brutal_kill, - type => worker, - modules => [Monitor] - } - ], SupFlags = #{ strategy => one_for_one, From 597bfbe310896649c2eb14c21cca07763425d309 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 15 Mar 2023 20:48:29 +0800 Subject: [PATCH 24/88] chore: update changes/ce/fix-10130.en.md Co-authored-by: Zaiming (Stone) Shi --- changes/ce/fix-10130.en.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/changes/ce/fix-10130.en.md b/changes/ce/fix-10130.en.md index d829e2c72..98484e38f 100644 --- a/changes/ce/fix-10130.en.md +++ b/changes/ce/fix-10130.en.md @@ -1,4 +1,3 @@ -Fix the EMQX node started by environment variable configuration cannot get the correct configuration information through HTTP API. -For example: `EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` -and the Statsd configuration information on Dashboard is garbled (not '127.0.0.1:8124'). -Related PR: [HOCON:234](https://github.com/emqx/hocon/pull/234). +Fix garbled config display in dashboard when the value is originally from environment variables. +For example, `env EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` results in unreadable string (not '127.0.0.1:8124') displayed in Dashboard's Statsd settings page. +Related PR: [HOCON#234](https://github.com/emqx/hocon/pull/234). From c1c38dd760dcbd8649e6af08ae76d448551b81b5 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 14:20:29 +0100 Subject: [PATCH 25/88] chore: cut.sh now exits with error if there are missing translations --- scripts/rel/cut.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 19a03a98d..08fa37192 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -233,10 +233,17 @@ if [ -d "${CHECKS_DIR}" ]; then fi generate_changelog () { - local from_tag="${PREV_TAG:-}" + local from_tag num_en num_zh + from_tag="${PREV_TAG:-}" if [[ -z $from_tag ]]; then from_tag="$(./scripts/find-prev-rel-tag.sh "$PROFILE")" fi + num_en=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.en.md') + num_zh=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.zh.md') + if [ "$num_en" -ne "$num_zh" ]; then + echo "Number of English and Chinese changelog files added since ${from_tag} do not match." + exit 1 + fi ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'en' -v "$TAG" > "changes/${TAG}.en.md" ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'zh' -v "$TAG" > "changes/${TAG}.zh.md" if [ -n "$(git diff --stat)" ]; then From 7f71ccbf25e5b23e062d1369b789a6e74b6031ce Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 14:22:19 +0100 Subject: [PATCH 26/88] ci: use aws-actions/configure-aws-credentials@v2 to address EOL of Node.js 12 --- .github/workflows/build_packages.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/upload-helm-charts.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f98e4a6dc..fe65ad455 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -339,7 +339,7 @@ jobs: echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 done cd - - - uses: aws-actions/configure-aws-credentials@v1-node16 + - uses: aws-actions/configure-aws-credentials@v2 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fe21545ca..31afba81d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false steps: - - uses: aws-actions/configure-aws-credentials@v1-node16 + - uses: aws-actions/configure-aws-credentials@v2 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/upload-helm-charts.yaml b/.github/workflows/upload-helm-charts.yaml index 319b50e24..d2b6723f3 100644 --- a/.github/workflows/upload-helm-charts.yaml +++ b/.github/workflows/upload-helm-charts.yaml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false steps: - - uses: aws-actions/configure-aws-credentials@v1-node16 + - uses: aws-actions/configure-aws-credentials@v2 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 7ef2c317e09fa53c22eba359db9517abfd68561d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 14:27:12 +0100 Subject: [PATCH 27/88] ci: use ubuntu22.04 everywhere --- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/run_fvt_tests.yaml | 2 +- .github/workflows/run_gitlint.yaml | 2 +- .github/workflows/run_jmeter_tests.yaml | 14 +++++++------- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 12 ++++++------ .github/workflows/shellcheck.yaml | 2 +- .github/workflows/stale.yaml | 2 +- .github/workflows/upload-helm-charts.yaml | 2 +- 13 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 13e26b204..52c467786 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -4,7 +4,7 @@ on: [pull_request] jobs: check_apps_version: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index f24e164d9..f42ada7ac 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: jobs: check_deps_integrity: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04 steps: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index a123ad93b..208911eb1 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -7,7 +7,7 @@ on: jobs: elixir_apps_check: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # just use the latest builder container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04" diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 348ed4931..e990f69dc 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,7 +7,7 @@ on: jobs: elixir_deps_check: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04 steps: diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 5517a2abc..eb25d57bd 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -11,7 +11,7 @@ on: jobs: elixir_release_build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: profile: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a95fcd805..edb582741 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -15,7 +15,7 @@ on: jobs: prepare: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-debian11 diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml index 9eb03c0b8..b89d0b7b0 100644 --- a/.github/workflows/run_gitlint.yaml +++ b/.github/workflows/run_gitlint.yaml @@ -4,7 +4,7 @@ on: [pull_request] jobs: run_gitlint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout source code uses: actions/checkout@v3 diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 3226f83a7..e402c7fed 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -10,7 +10,7 @@ on: jobs: build_emqx_for_jmeter_tests: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: version: ${{ steps.build_docker.outputs.version}} steps: @@ -44,7 +44,7 @@ jobs: path: ./emqx.tar advanced_feat: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -136,7 +136,7 @@ jobs: path: ./jmeter_logs pgsql_authn_authz: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -245,7 +245,7 @@ jobs: path: ./jmeter_logs mysql_authn_authz: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -351,7 +351,7 @@ jobs: path: ./jmeter_logs JWT_authn: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -449,7 +449,7 @@ jobs: path: ./jmeter_logs built_in_database_authn_authz: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -541,7 +541,7 @@ jobs: path: ./jmeter_logs delete-artifact: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz] steps: - uses: geekyeggo/delete-artifact@v2 diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index ca3e0e0ce..1e14eb2b3 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -58,7 +58,7 @@ jobs: needs: - relup_test_plan if: needs.relup_test_plan.outputs.OLD_VERSIONS != '[]' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 5006fe760..e76b05f7e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -16,7 +16,7 @@ on: jobs: build-matrix: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: prepare: ${{ steps.matrix.outputs.prepare }} host: ${{ steps.matrix.outputs.host }} @@ -63,7 +63,7 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 @@ -100,7 +100,7 @@ jobs: defaults: run: shell: bash - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 @@ -156,7 +156,7 @@ jobs: - name: run tests working-directory: source env: - DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" + DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" MONGO_TAG: "5" MYSQL_TAG: "8" PGSQL_TAG: "13" @@ -186,7 +186,7 @@ jobs: matrix: include: ${{ fromJson(needs.build-matrix.outputs.host) }} - container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" defaults: run: shell: bash @@ -262,7 +262,7 @@ jobs: # do this in a separate job upload_coverdata: needs: make_cover - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Coveralls Finished env: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 558ecf3bf..7f29572b9 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -5,7 +5,7 @@ on: jobs: shellcheck: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout source code uses: actions/checkout@v3 diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index cf6229b13..6b67c6f3b 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -10,7 +10,7 @@ on: jobs: stale: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: github.repository_owner == 'emqx' permissions: issues: write diff --git a/.github/workflows/upload-helm-charts.yaml b/.github/workflows/upload-helm-charts.yaml index d2b6723f3..4b18efef8 100644 --- a/.github/workflows/upload-helm-charts.yaml +++ b/.github/workflows/upload-helm-charts.yaml @@ -11,7 +11,7 @@ on: jobs: upload: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false steps: From 0deb9925c13bc93803ac72c26b383a88637fdf23 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Mar 2023 10:28:48 -0300 Subject: [PATCH 28/88] docs: improve descriptions Co-authored-by: William Yang --- apps/emqx/i18n/emqx_schema_i18n.conf | 6 +++--- changes/ce/feat-10067.zh.md | 1 - changes/ce/{feat-10067.en.md => feat-10128.en.md} | 0 changes/ce/feat-10128.zh.md | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 changes/ce/feat-10067.zh.md rename changes/ce/{feat-10067.en.md => feat-10128.en.md} (100%) create mode 100644 changes/ce/feat-10128.zh.md diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index ce76aff3c..c0fb4f07e 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1795,18 +1795,18 @@ server_ssl_opts_schema_ocsp_refresh_interval { } label: { en: "OCSP Refresh Interval" - zh: "OCSP刷新间隔" + zh: "OCSP 刷新间隔" } } server_ssl_opts_schema_ocsp_refresh_http_timeout { desc { en: "The timeout for the HTTP request when checking OCSP responses." - zh: "检查OCSP响应时，HTTP请求的超时。" + zh: "检查 OCSP 响应时，HTTP 请求的超时。" } label: { en: "OCSP Refresh HTTP Timeout" - zh: "OCSP刷新HTTP超时" + zh: "OCSP 刷新 HTTP 超时" } } diff --git a/changes/ce/feat-10067.zh.md b/changes/ce/feat-10067.zh.md deleted file mode 100644 index d0efe4d5b..000000000 --- a/changes/ce/feat-10067.zh.md +++ /dev/null @@ -1 +0,0 @@ -为SSL MQTT监听器增加对OCSP订书和CRL检查的支持。 diff --git a/changes/ce/feat-10067.en.md b/changes/ce/feat-10128.en.md similarity index 100% rename from changes/ce/feat-10067.en.md rename to changes/ce/feat-10128.en.md diff --git a/changes/ce/feat-10128.zh.md b/changes/ce/feat-10128.zh.md new file mode 100644 index 000000000..544ea400e --- /dev/null +++ b/changes/ce/feat-10128.zh.md @@ -0,0 +1 @@ +为 SSL MQTT 监听器增加对 OCSP Stapling 和 CRL 检查的支持。 From cad6492c990c4f694878fda8a1b64ed3bc776741 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Mar 2023 14:51:28 +0300 Subject: [PATCH 29/88] perf(bridge-api): ask bridge listings in parallel Also rename response formatting functions to better clarify their purpose. --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_bridge/src/emqx_bridge_api.erl | 34 +++-- .../src/proto/emqx_bridge_proto_v2.erl | 4 + .../src/proto/emqx_bridge_proto_v3.erl | 128 ++++++++++++++++++ apps/emqx_resource/src/emqx_resource.erl | 2 + 5 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 769145722..904611199 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -4,6 +4,7 @@ {emqx_authz,1}. {emqx_bridge,1}. {emqx_bridge,2}. +{emqx_bridge,3}. {emqx_broker,1}. {emqx_cm,1}. {emqx_conf,1}. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index ff55976d0..e9f31d010 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -483,11 +483,18 @@ schema("/bridges_probe") -> end end; '/bridges'(get, _Params) -> - {200, - zip_bridges([ - [format_resp(Data, Node) || Data <- emqx_bridge_proto_v1:list_bridges(Node)] - || Node <- mria:running_nodes() - ])}. + Nodes = mria:running_nodes(), + NodeReplies = emqx_bridge_proto_v3:list_bridges_on_nodes(Nodes), + case is_ok(NodeReplies) of + {ok, NodeBridges} -> + AllBridges = [ + format_resource(Data, Node) + || {Node, Bridges} <- lists:zip(Nodes, NodeBridges), Data <- Bridges + ], + {200, zip_bridges([AllBridges])}; + {error, Reason} -> + {500, error_msg('INTERNAL_ERROR', Reason)} + end. '/bridges/:id'(get, #{bindings := #{id := Id}}) -> ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); @@ -591,7 +598,7 @@ lookup_from_all_nodes_metrics(BridgeType, BridgeName, SuccCode) -> do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) -> Nodes = mria:running_nodes(), - case is_ok(emqx_bridge_proto_v1:lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of + case is_ok(emqx_bridge_proto_v3:lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of {ok, [{ok, _} | _] = Results} -> {SuccCode, FormatFun([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> @@ -602,7 +609,7 @@ do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) -> lookup_from_local_node(BridgeType, BridgeName) -> case emqx_bridge:lookup(BridgeType, BridgeName) of - {ok, Res} -> {ok, format_resp(Res)}; + {ok, Res} -> {ok, format_resource(Res, node())}; Error -> Error end. @@ -802,10 +809,7 @@ aggregate_metrics( aggregate_metrics(#{}, Metrics) -> Metrics. -format_resp(Data) -> - format_resp(Data, node()). - -format_resp( +format_resource( #{ type := Type, name := BridgeName, @@ -974,7 +978,7 @@ do_bpapi_call(Node, Call, Args) -> do_bpapi_call_vsn(SupportedVersion, Call, Args) -> case lists:member(SupportedVersion, supported_versions(Call)) of true -> - apply(emqx_bridge_proto_v2, Call, Args); + apply(emqx_bridge_proto_v3, Call, Args); false -> {error, not_implemented} end. @@ -984,9 +988,9 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> [2]; -supported_versions(start_bridges_to_all_nodes) -> [2]; -supported_versions(_Call) -> [1, 2]. +supported_versions(start_bridge_to_node) -> [2, 3]; +supported_versions(start_bridges_to_all_nodes) -> [2, 3]; +supported_versions(_Call) -> [1, 2, 3]. to_hr_reason(nxdomain) -> <<"Host not found">>; diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl index 0fd733380..bcf6ca198 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v2.erl @@ -20,6 +20,7 @@ -export([ introduced_in/0, + deprecated_since/0, list_bridges/1, restart_bridge_to_node/3, @@ -38,6 +39,9 @@ introduced_in() -> "5.0.17". +deprecated_since() -> + "5.0.21". + -spec list_bridges(node()) -> list() | emqx_rpc:badrpc(). list_bridges(Node) -> rpc:call(Node, emqx_bridge, list, [], ?TIMEOUT). diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl new file mode 100644 index 000000000..a35db5d96 --- /dev/null +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl @@ -0,0 +1,128 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + list_bridges/1, + list_bridges_on_nodes/1, + restart_bridge_to_node/3, + start_bridge_to_node/3, + stop_bridge_to_node/3, + lookup_from_all_nodes/3, + restart_bridges_to_all_nodes/3, + start_bridges_to_all_nodes/3, + stop_bridges_to_all_nodes/3 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.0.21". + +-spec list_bridges(node()) -> list() | emqx_rpc:badrpc(). +list_bridges(Node) -> + rpc:call(Node, emqx_bridge, list, [], ?TIMEOUT). + +-spec list_bridges_on_nodes([node()]) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +list_bridges_on_nodes(Nodes) -> + erpc:multicall(Nodes, emqx_bridge, list, [], ?TIMEOUT). + +-type key() :: atom() | binary() | [byte()]. + +-spec restart_bridge_to_node(node(), key(), key()) -> + term(). +restart_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridge_to_node(node(), key(), key()) -> + term(). +start_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridge_to_node(node(), key(), key()) -> + term(). +stop_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec restart_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec lookup_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + lookup_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 2c6865e04..57d56b339 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -112,6 +112,8 @@ -export([apply_reply_fun/2]). +-export_type([resource_data/0]). + -optional_callbacks([ on_query/3, on_batch_query/3, From b3e7e51094daf95d5ad3cee60db920300f793f11 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Mar 2023 18:12:41 +0300 Subject: [PATCH 30/88] test(bridge): drop unnecessary cleanup routines Since `end_per_testcase` cleans out all the resources anyway. --- .../emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index e2c9382db..bd6af9323 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -import(emqx_dashboard_api_test_helpers, [request/4, uri/1]). --import(emqx_common_test_helpers, [on_exit/1]). -include("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -164,9 +163,9 @@ init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), ok = snabbkaffe:start_trace(), Config. + end_per_testcase(_, _Config) -> clear_resources(), - emqx_common_test_helpers:call_janitor(), snabbkaffe:stop(), ok. @@ -710,13 +709,6 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> } ), - on_exit(fun() -> - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - ok - end), - %% we now test if the bridge works as expected LocalTopic = <>, RemoteTopic = <>, @@ -827,13 +819,6 @@ t_mqtt_conn_bridge_egress_async_reconnect(_) -> } ), - on_exit(fun() -> - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - ok - end), - Self = self(), LocalTopic = <>, RemoteTopic = <>, From e411c5d5f8cb357e48ba9fa33ae713ec69f07019 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 13 Mar 2023 18:28:28 +0300 Subject: [PATCH 31/88] refactor(resman): work with state cache atomically Also ensure that cache entries are always consistent with `Data`, so that most of the code could rely on reading the cached entry most of the time. --- .../src/emqx_resource_manager.erl | 162 +++--- .../test/emqx_connector_demo.erl | 12 +- .../test/emqx_resource_SUITE.erl | 528 +++++++++++------- 3 files changed, 423 insertions(+), 279 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 05d100913..0983dff8d 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -18,6 +18,7 @@ -include("emqx_resource.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). % API -export([ @@ -303,26 +304,30 @@ start_link(MgrId, ResId, Group, ResourceType, Config, Opts) -> query_mode = maps:get(query_mode, Opts, sync), config = Config, opts = Opts, - status = connecting, state = undefined, error = undefined }, gen_statem:start_link(?MODULE, {Data, Opts}, []). -init({Data, Opts}) -> +init({DataIn, Opts}) -> process_flag(trap_exit, true), - %% init the cache so that lookup/1 will always return something - DataWithPid = Data#data{pid = self()}, - insert_cache(DataWithPid#data.id, DataWithPid#data.group, DataWithPid), + Data = DataIn#data{pid = self()}, case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of - true -> {ok, connecting, DataWithPid, {next_event, internal, start_resource}}; - false -> {ok, stopped, DataWithPid} + true -> + %% init the cache so that lookup/1 will always return something + UpdatedData = update_state(Data#data{status = connecting}), + {ok, connecting, UpdatedData, {next_event, internal, start_resource}}; + false -> + %% init the cache so that lookup/1 will always return something + UpdatedData = update_state(Data#data{status = stopped}), + {ok, stopped, UpdatedData} end. +terminate({shutdown, removed}, _State, _Data) -> + ok; terminate(_Reason, _State, Data) -> - _ = stop_resource(Data), - _ = maybe_clear_alarm(Data#data.id), - delete_cache(Data#data.id, Data#data.manager_id), + _ = maybe_stop_resource(Data), + ok = delete_cache(Data#data.id, Data#data.manager_id), ok. %% Behavior callback @@ -333,11 +338,12 @@ callback_mode() -> [handle_event_function, state_enter]. % Called during testing to force a specific state handle_event({call, From}, set_resource_status_connecting, _State, Data) -> - {next_state, connecting, Data#data{status = connecting}, [{reply, From, ok}]}; + UpdatedData = update_state(Data#data{status = connecting}, Data), + {next_state, connecting, UpdatedData, [{reply, From, ok}]}; % Called when the resource is to be restarted handle_event({call, From}, restart, _State, Data) -> - _ = stop_resource(Data), - start_resource(Data, From); + DataNext = stop_resource(Data), + start_resource(DataNext, From); % Called when the resource is to be started (also used for manual reconnect) handle_event({call, From}, start, State, Data) when State =:= stopped orelse @@ -347,16 +353,14 @@ handle_event({call, From}, start, State, Data) when handle_event({call, From}, start, _State, _Data) -> {keep_state_and_data, [{reply, From, ok}]}; % Called when the resource received a `quit` message -handle_event(info, quit, stopped, _Data) -> - {stop, {shutdown, quit}}; handle_event(info, quit, _State, _Data) -> {stop, {shutdown, quit}}; % Called when the resource is to be stopped handle_event({call, From}, stop, stopped, _Data) -> {keep_state_and_data, [{reply, From, ok}]}; handle_event({call, From}, stop, _State, Data) -> - Result = stop_resource(Data), - {next_state, stopped, Data, [{reply, From, Result}]}; + UpdatedData = stop_resource(Data), + {next_state, stopped, update_state(UpdatedData, Data), [{reply, From, ok}]}; % Called when a resource is to be stopped and removed. handle_event({call, From}, {remove, ClearMetrics}, _State, Data) -> handle_remove_event(From, ClearMetrics, Data); @@ -371,11 +375,9 @@ handle_event({call, From}, health_check, stopped, _Data) -> handle_event({call, From}, health_check, _State, Data) -> handle_manually_health_check(From, Data); % State: CONNECTING -handle_event(enter, _OldState, connecting, Data) -> - UpdatedData = Data#data{status = connecting}, - insert_cache(Data#data.id, Data#data.group, Data), - Actions = [{state_timeout, 0, health_check}], - {keep_state, UpdatedData, Actions}; +handle_event(enter, _OldState, connecting = State, Data) -> + ok = log_state_consistency(State, Data), + {keep_state_and_data, [{state_timeout, 0, health_check}]}; handle_event(internal, start_resource, connecting, Data) -> start_resource(Data, undefined); handle_event(state_timeout, health_check, connecting, Data) -> @@ -383,27 +385,23 @@ handle_event(state_timeout, health_check, connecting, Data) -> %% State: CONNECTED %% The connected state is entered after a successful on_start/2 of the callback mod %% and successful health_checks -handle_event(enter, _OldState, connected, Data) -> - UpdatedData = Data#data{status = connected}, - insert_cache(Data#data.id, Data#data.group, UpdatedData), +handle_event(enter, _OldState, connected = State, Data) -> + ok = log_state_consistency(State, Data), _ = emqx_alarm:deactivate(Data#data.id), - Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}], - {next_state, connected, UpdatedData, Actions}; + {keep_state_and_data, health_check_actions(Data)}; handle_event(state_timeout, health_check, connected, Data) -> handle_connected_health_check(Data); %% State: DISCONNECTED -handle_event(enter, _OldState, disconnected, Data) -> - UpdatedData = Data#data{status = disconnected}, - insert_cache(Data#data.id, Data#data.group, UpdatedData), - handle_disconnected_state_enter(UpdatedData); +handle_event(enter, _OldState, disconnected = State, Data) -> + ok = log_state_consistency(State, Data), + {keep_state_and_data, retry_actions(Data)}; handle_event(state_timeout, auto_retry, disconnected, Data) -> start_resource(Data, undefined); %% State: STOPPED %% The stopped state is entered after the resource has been explicitly stopped -handle_event(enter, _OldState, stopped, Data) -> - UpdatedData = Data#data{status = stopped}, - insert_cache(Data#data.id, Data#data.group, UpdatedData), - {next_state, stopped, UpdatedData}; +handle_event(enter, _OldState, stopped = State, Data) -> + ok = log_state_consistency(State, Data), + {keep_state_and_data, []}; % Ignore all other events handle_event(EventType, EventData, State, Data) -> ?SLOG( @@ -418,6 +416,22 @@ handle_event(EventType, EventData, State, Data) -> ), keep_state_and_data. +log_state_consistency(State, #data{status = State} = Data) -> + log_cache_consistency(read_cache(Data#data.id), Data); +log_state_consistency(State, Data) -> + ?tp(warning, "inconsistent_state", #{ + state => State, + data => Data + }). + +log_cache_consistency({_, Data}, Data) -> + ok; +log_cache_consistency({_, DataCached}, Data) -> + ?tp(warning, "inconsistent_cache", #{ + cache => DataCached, + data => Data + }). + %%------------------------------------------------------------------------------ %% internal functions %%------------------------------------------------------------------------------ @@ -451,10 +465,12 @@ delete_cache(ResId, MgrId) -> end. do_delete_cache(<> = ResId) -> - ets:delete(?ETS_TABLE, {owner, ResId}), - ets:delete(?ETS_TABLE, ResId); + true = ets:delete(?ETS_TABLE, {owner, ResId}), + true = ets:delete(?ETS_TABLE, ResId), + ok; do_delete_cache(ResId) -> - ets:delete(?ETS_TABLE, ResId). + true = ets:delete(?ETS_TABLE, ResId), + ok. set_new_owner(ResId) -> MgrId = make_manager_id(ResId), @@ -471,9 +487,6 @@ get_owner(ResId) -> [] -> not_found end. -handle_disconnected_state_enter(Data) -> - {next_state, disconnected, Data, retry_actions(Data)}. - retry_actions(Data) -> case maps:get(auto_restart_interval, Data#data.opts, ?AUTO_RESTART_INTERVAL) of undefined -> @@ -482,24 +495,27 @@ retry_actions(Data) -> [{state_timeout, RetryInterval, auto_retry}] end. +health_check_actions(Data) -> + [{state_timeout, health_check_interval(Data#data.opts), health_check}]. + handle_remove_event(From, ClearMetrics, Data) -> - stop_resource(Data), + _ = stop_resource(Data), + ok = delete_cache(Data#data.id, Data#data.manager_id), ok = emqx_resource_buffer_worker_sup:stop_workers(Data#data.id, Data#data.opts), case ClearMetrics of true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok end, - {stop_and_reply, normal, [{reply, From, ok}]}. + {stop_and_reply, {shutdown, removed}, [{reply, From, ok}]}. start_resource(Data, From) -> %% in case the emqx_resource:call_start/2 hangs, the lookup/1 can read status from the cache - insert_cache(Data#data.id, Data#data.group, Data), case emqx_resource:call_start(Data#data.manager_id, Data#data.mod, Data#data.config) of {ok, ResourceState} -> - UpdatedData = Data#data{state = ResourceState, status = connecting}, + UpdatedData = Data#data{status = connecting, state = ResourceState}, %% Perform an initial health_check immediately before transitioning into a connected state Actions = maybe_reply([{state_timeout, 0, health_check}], From, ok), - {next_state, connecting, UpdatedData, Actions}; + {next_state, connecting, update_state(UpdatedData, Data), Actions}; {error, Reason} = Err -> ?SLOG(warning, #{ msg => start_resource_failed, @@ -509,34 +525,42 @@ start_resource(Data, From) -> _ = maybe_alarm(disconnected, Data#data.id), %% Keep track of the error reason why the connection did not work %% so that the Reason can be returned when the verification call is made. - UpdatedData = Data#data{error = Reason}, + UpdatedData = Data#data{status = disconnected, error = Reason}, Actions = maybe_reply(retry_actions(UpdatedData), From, Err), - {next_state, disconnected, UpdatedData, Actions} + {next_state, disconnected, update_state(UpdatedData, Data), Actions} end. -stop_resource(#data{state = undefined, id = ResId} = _Data) -> - _ = maybe_clear_alarm(ResId), - ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId), - ok; -stop_resource(Data) -> +maybe_stop_resource(#data{status = Status} = Data) when Status /= stopped -> + stop_resource(Data); +maybe_stop_resource(#data{status = stopped} = Data) -> + Data. + +stop_resource(#data{state = ResState, id = ResId} = Data) -> %% We don't care the return value of the Mod:on_stop/2. %% The callback mod should make sure the resource is stopped after on_stop/2 %% is returned. - ResId = Data#data.id, - _ = emqx_resource:call_stop(Data#data.manager_id, Data#data.mod, Data#data.state), + case ResState /= undefined of + true -> + emqx_resource:call_stop(Data#data.manager_id, Data#data.mod, ResState); + false -> + ok + end, _ = maybe_clear_alarm(ResId), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId), - ok. + Data#data{status = stopped}. make_test_id() -> RandId = iolist_to_binary(emqx_misc:gen_id(16)), <>. handle_manually_health_check(From, Data) -> - with_health_check(Data, fun(Status, UpdatedData) -> - Actions = [{reply, From, {ok, Status}}], - {next_state, Status, UpdatedData, Actions} - end). + with_health_check( + Data, + fun(Status, UpdatedData) -> + Actions = [{reply, From, {ok, Status}}], + {next_state, Status, UpdatedData, Actions} + end + ). handle_connecting_health_check(Data) -> with_health_check( @@ -545,8 +569,7 @@ handle_connecting_health_check(Data) -> (connected, UpdatedData) -> {next_state, connected, UpdatedData}; (connecting, UpdatedData) -> - Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}], - {keep_state, UpdatedData, Actions}; + {keep_state, UpdatedData, health_check_actions(UpdatedData)}; (disconnected, UpdatedData) -> {next_state, disconnected, UpdatedData} end @@ -557,8 +580,7 @@ handle_connected_health_check(Data) -> Data, fun (connected, UpdatedData) -> - Actions = [{state_timeout, health_check_interval(Data#data.opts), health_check}], - {keep_state, UpdatedData, Actions}; + {keep_state, UpdatedData, health_check_actions(UpdatedData)}; (Status, UpdatedData) -> ?SLOG(warning, #{ msg => health_check_failed, @@ -580,8 +602,16 @@ with_health_check(Data, Func) -> UpdatedData = Data#data{ state = NewState, status = Status, error = Err }, - insert_cache(ResId, UpdatedData#data.group, UpdatedData), - Func(Status, UpdatedData). + Func(Status, update_state(UpdatedData, Data)). + +update_state(Data) -> + update_state(Data, undefined). + +update_state(DataWas, DataWas) -> + DataWas; +update_state(Data, _DataWas) -> + _ = insert_cache(Data#data.id, Data#data.group, Data), + Data. health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index f41087b20..a863dbb78 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -75,8 +75,7 @@ on_start(InstId, #{name := Name} = Opts) -> on_stop(_InstId, #{stop_error := true}) -> {error, stop_error}; on_stop(_InstId, #{pid := Pid}) -> - erlang:exit(Pid, shutdown), - ok. + stop_counter_process(Pid). on_query(_InstId, get_state, State) -> {ok, State}; @@ -247,6 +246,15 @@ spawn_counter_process(Name, Register) -> true = maybe_register(Name, Pid, Register), Pid. +stop_counter_process(Pid) -> + true = erlang:is_process_alive(Pid), + true = erlang:exit(Pid, shutdown), + receive + {'EXIT', Pid, shutdown} -> ok + after 5000 -> + {error, timeout} + end. + counter_loop() -> counter_loop(#{ counter => 0, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index af72e86f9..f6d2b7ab4 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -72,115 +72,156 @@ t_check_config(_) -> {error, _} = emqx_resource:check_config(?TEST_RESOURCE, #{invalid => config}). t_create_remove(_) -> - {error, _} = emqx_resource:check_and_create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{unknown => test_resource} - ), + ?check_trace( + begin + ?assertMatch( + {error, _}, + emqx_resource:check_and_create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{unknown => test_resource} + ) + ), - {ok, _} = emqx_resource:create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource} - ), + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource} + ) + ), - {ok, _} = emqx_resource:recreate( - ?ID, - ?TEST_RESOURCE, - #{name => test_resource}, - #{} - ), - {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), + ?assertMatch( + {ok, _}, + emqx_resource:recreate( + ?ID, + ?TEST_RESOURCE, + #{name => test_resource}, + #{} + ) + ), - ?assert(is_process_alive(Pid)), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), - ok = emqx_resource:remove(?ID), - {error, _} = emqx_resource:remove(?ID), + ?assert(is_process_alive(Pid)), - ?assertNot(is_process_alive(Pid)). + ?assertEqual(ok, emqx_resource:remove(?ID)), + ?assertMatch({error, _}, emqx_resource:remove(?ID)), + + ?assertNot(is_process_alive(Pid)) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_create_remove_local(_) -> - {error, _} = emqx_resource:check_and_create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{unknown => test_resource} - ), + ?check_trace( + begin + ?assertMatch( + {error, _}, + emqx_resource:check_and_create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{unknown => test_resource} + ) + ), - {ok, _} = emqx_resource:create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource} - ), + ?assertMatch( + {ok, _}, + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource} + ) + ), - emqx_resource:recreate_local( - ?ID, - ?TEST_RESOURCE, - #{name => test_resource}, - #{} - ), - {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), + emqx_resource:recreate_local( + ?ID, + ?TEST_RESOURCE, + #{name => test_resource}, + #{} + ), - ?assert(is_process_alive(Pid)), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), - emqx_resource:set_resource_status_connecting(?ID), + ?assert(is_process_alive(Pid)), - emqx_resource:recreate_local( - ?ID, - ?TEST_RESOURCE, - #{name => test_resource}, - #{} - ), + emqx_resource:set_resource_status_connecting(?ID), - ok = emqx_resource:remove_local(?ID), - {error, _} = emqx_resource:remove_local(?ID), + emqx_resource:recreate_local( + ?ID, + ?TEST_RESOURCE, + #{name => test_resource}, + #{} + ), - ?assertMatch( - ?RESOURCE_ERROR(not_found), - emqx_resource:query(?ID, get_state) - ), - ?assertNot(is_process_alive(Pid)). + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + ?assertMatch({error, _}, emqx_resource:remove_local(?ID)), + + ?assertMatch( + ?RESOURCE_ERROR(not_found), + emqx_resource:query(?ID, get_state) + ), + + ?assertNot(is_process_alive(Pid)) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_do_not_start_after_created(_) -> - ct:pal("creating resource"), - {ok, _} = emqx_resource:create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource}, - #{start_after_created => false} - ), - %% the resource should remain `disconnected` after created - timer:sleep(200), - ?assertMatch( - ?RESOURCE_ERROR(stopped), - emqx_resource:query(?ID, get_state) - ), - ?assertMatch( - {ok, _, #{status := stopped}}, - emqx_resource:get_instance(?ID) - ), + ?check_trace( + begin + ?assertMatch( + {ok, _}, + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{start_after_created => false} + ) + ), + %% the resource should remain `disconnected` after created + timer:sleep(200), + ?assertMatch( + ?RESOURCE_ERROR(stopped), + emqx_resource:query(?ID, get_state) + ), + ?assertMatch( + {ok, _, #{status := stopped}}, + emqx_resource:get_instance(?ID) + ), - %% start the resource manually.. - ct:pal("starting resource manually"), - ok = emqx_resource:start(?ID), - {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), - ?assert(is_process_alive(Pid)), + %% start the resource manually.. + ?assertEqual(ok, emqx_resource:start(?ID)), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), + ?assert(is_process_alive(Pid)), - %% restart the resource - ct:pal("restarting resource"), - ok = emqx_resource:restart(?ID), - ?assertNot(is_process_alive(Pid)), - {ok, #{pid := Pid2}} = emqx_resource:query(?ID, get_state), - ?assert(is_process_alive(Pid2)), + %% restart the resource + ?assertEqual(ok, emqx_resource:restart(?ID)), + ?assertNot(is_process_alive(Pid)), + {ok, #{pid := Pid2}} = emqx_resource:query(?ID, get_state), + ?assert(is_process_alive(Pid2)), - ct:pal("removing resource"), - ok = emqx_resource:remove_local(?ID), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), - ?assertNot(is_process_alive(Pid2)). + ?assertNot(is_process_alive(Pid2)) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_query(_) -> {ok, _} = emqx_resource:create_local( @@ -771,153 +812,210 @@ t_query_counter_async_inflight_batch(_) -> ok = emqx_resource:remove_local(?ID). t_healthy_timeout(_) -> - {ok, _} = emqx_resource:create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => <<"bad_not_atom_name">>, register => true}, - %% the ?TEST_RESOURCE always returns the `Mod:on_get_status/2` 300ms later. - #{health_check_interval => 200} - ), - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - emqx_resource:query(?ID, get_state, #{timeout => 1_000}) - ), - ?assertMatch({ok, _Group, #{status := disconnected}}, emqx_resource_manager:ets_lookup(?ID)), - ok = emqx_resource:remove_local(?ID). + ?check_trace( + begin + ?assertMatch( + {ok, _}, + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => <<"bad_not_atom_name">>, register => true}, + %% the ?TEST_RESOURCE always returns the `Mod:on_get_status/2` 300ms later. + #{health_check_interval => 200} + ) + ), + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + emqx_resource:query(?ID, get_state, #{timeout => 1_000}) + ), + ?assertMatch( + {ok, _Group, #{status := disconnected}}, emqx_resource_manager:lookup(?ID) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_healthy(_) -> - {ok, _} = emqx_resource:create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource} - ), - {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), - timer:sleep(300), - emqx_resource:set_resource_status_connecting(?ID), + ?check_trace( + begin + ?assertMatch( + {ok, _}, + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource} + ) + ), + {ok, #{pid := Pid}} = emqx_resource:query(?ID, get_state), + timer:sleep(300), + emqx_resource:set_resource_status_connecting(?ID), - {ok, connected} = emqx_resource:health_check(?ID), - ?assertMatch( - [#{status := connected}], - emqx_resource:list_instances_verbose() - ), + ?assertEqual({ok, connected}, emqx_resource:health_check(?ID)), + ?assertMatch( + [#{status := connected}], + emqx_resource:list_instances_verbose() + ), - erlang:exit(Pid, shutdown), + erlang:exit(Pid, shutdown), - ?assertEqual({ok, disconnected}, emqx_resource:health_check(?ID)), + ?assertEqual({ok, disconnected}, emqx_resource:health_check(?ID)), - ?assertMatch( - [#{status := disconnected}], - emqx_resource:list_instances_verbose() - ), + ?assertMatch( + [#{status := disconnected}], + emqx_resource:list_instances_verbose() + ), - ok = emqx_resource:remove_local(?ID). + ?assertEqual(ok, emqx_resource:remove_local(?ID)) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_stop_start(_) -> - {error, _} = emqx_resource:check_and_create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{unknown => test_resource} - ), + ?check_trace( + begin + ?assertMatch( + {error, _}, + emqx_resource:check_and_create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{unknown => test_resource} + ) + ), - {ok, _} = emqx_resource:check_and_create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{<<"name">> => <<"test_resource">>} - ), + ?assertMatch( + {ok, _}, + emqx_resource:check_and_create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{<<"name">> => <<"test_resource">>} + ) + ), - %% add some metrics to test their persistence - WorkerID0 = <<"worker:0">>, - WorkerID1 = <<"worker:1">>, - emqx_resource_metrics:inflight_set(?ID, WorkerID0, 2), - emqx_resource_metrics:inflight_set(?ID, WorkerID1, 3), - ?assertEqual(5, emqx_resource_metrics:inflight_get(?ID)), + %% add some metrics to test their persistence + WorkerID0 = <<"worker:0">>, + WorkerID1 = <<"worker:1">>, + emqx_resource_metrics:inflight_set(?ID, WorkerID0, 2), + emqx_resource_metrics:inflight_set(?ID, WorkerID1, 3), + ?assertEqual(5, emqx_resource_metrics:inflight_get(?ID)), - {ok, _} = emqx_resource:check_and_recreate( - ?ID, - ?TEST_RESOURCE, - #{<<"name">> => <<"test_resource">>}, - #{} - ), + ?assertMatch( + {ok, _}, + emqx_resource:check_and_recreate( + ?ID, + ?TEST_RESOURCE, + #{<<"name">> => <<"test_resource">>}, + #{} + ) + ), - {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state), - ?assert(is_process_alive(Pid0)), + ?assert(is_process_alive(Pid0)), - %% metrics are reset when recreating - %% depending on timing, might show the request we just did. - ct:sleep(500), - ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), + %% metrics are reset when recreating + %% depending on timing, might show the request we just did. + ct:sleep(500), + ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), - ok = emqx_resource:stop(?ID), + ok = emqx_resource:stop(?ID), - ?assertNot(is_process_alive(Pid0)), + ?assertNot(is_process_alive(Pid0)), - ?assertMatch( - ?RESOURCE_ERROR(stopped), - emqx_resource:query(?ID, get_state) - ), + ?assertMatch( + ?RESOURCE_ERROR(stopped), + emqx_resource:query(?ID, get_state) + ), - ok = emqx_resource:restart(?ID), - timer:sleep(300), + ?assertEqual(ok, emqx_resource:restart(?ID)), + timer:sleep(300), - {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state), - ?assert(is_process_alive(Pid1)), + ?assert(is_process_alive(Pid1)), - %% now stop while resetting the metrics - ct:sleep(500), - emqx_resource_metrics:inflight_set(?ID, WorkerID0, 1), - emqx_resource_metrics:inflight_set(?ID, WorkerID1, 4), - ?assertEqual(5, emqx_resource_metrics:inflight_get(?ID)), - ok = emqx_resource:stop(?ID), - ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)), + %% now stop while resetting the metrics + ct:sleep(500), + emqx_resource_metrics:inflight_set(?ID, WorkerID0, 1), + emqx_resource_metrics:inflight_set(?ID, WorkerID1, 4), + ?assertEqual(5, emqx_resource_metrics:inflight_get(?ID)), + ?assertEqual(ok, emqx_resource:stop(?ID)), + ?assertEqual(0, emqx_resource_metrics:inflight_get(?ID)) + end, - ok. + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_stop_start_local(_) -> - {error, _} = emqx_resource:check_and_create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{unknown => test_resource} - ), + ?check_trace( + begin + ?assertMatch( + {error, _}, + emqx_resource:check_and_create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{unknown => test_resource} + ) + ), - {ok, _} = emqx_resource:check_and_create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{<<"name">> => <<"test_resource">>} - ), + ?assertMatch( + {ok, _}, + emqx_resource:check_and_create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{<<"name">> => <<"test_resource">>} + ) + ), - {ok, _} = emqx_resource:check_and_recreate_local( - ?ID, - ?TEST_RESOURCE, - #{<<"name">> => <<"test_resource">>}, - #{} - ), + ?assertMatch( + {ok, _}, + emqx_resource:check_and_recreate_local( + ?ID, + ?TEST_RESOURCE, + #{<<"name">> => <<"test_resource">>}, + #{} + ) + ), - {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid0}} = emqx_resource:query(?ID, get_state), - ?assert(is_process_alive(Pid0)), + ?assert(is_process_alive(Pid0)), - ok = emqx_resource:stop(?ID), + ?assertEqual(ok, emqx_resource:stop(?ID)), - ?assertNot(is_process_alive(Pid0)), + ?assertNot(is_process_alive(Pid0)), - ?assertMatch( - ?RESOURCE_ERROR(stopped), - emqx_resource:query(?ID, get_state) - ), + ?assertMatch( + ?RESOURCE_ERROR(stopped), + emqx_resource:query(?ID, get_state) + ), - ok = emqx_resource:restart(?ID), + ?assertEqual(ok, emqx_resource:restart(?ID)), - {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state), + {ok, #{pid := Pid1}} = emqx_resource:query(?ID, get_state), - ?assert(is_process_alive(Pid1)). + ?assert(is_process_alive(Pid1)) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end + ). t_list_filter(_) -> {ok, _} = emqx_resource:create_local( @@ -1031,16 +1129,24 @@ t_auto_retry(_) -> ?assertEqual(ok, Res). t_health_check_disconnected(_) -> - _ = emqx_resource:create_local( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource, create_error => true}, - #{auto_retry_interval => 100} - ), - ?assertEqual( - {ok, disconnected}, - emqx_resource:health_check(?ID) + ?check_trace( + begin + _ = emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource, create_error => true}, + #{auto_retry_interval => 100} + ), + ?assertEqual( + {ok, disconnected}, + emqx_resource:health_check(?ID) + ) + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("inconsistent_state", Trace)), + ?assertEqual([], ?of_kind("inconsistent_cache", Trace)) + end ). t_unblock_only_required_buffer_workers(_) -> From 29907875bf11602e5edfa282c7a8d4ed8ca73839 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 14 Mar 2023 13:30:24 +0300 Subject: [PATCH 32/88] test(bufworker): set `batch_time` for batch-related testcases By default it's `0` since e9d3fc51. This made a couple of tests prone to flapping. --- .../emqx_resource/test/emqx_resource_SUITE.erl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index f6d2b7ab4..ff7e1d347 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -263,7 +263,11 @@ t_batch_query_counter(_) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource, register => true}, - #{batch_size => BatchSize, query_mode => sync} + #{ + batch_size => BatchSize, + batch_time => 100, + query_mode => sync + } ), ?check_trace( @@ -622,6 +626,7 @@ t_query_counter_async_inflight_batch(_) -> #{ query_mode => async, batch_size => BatchSize, + batch_time => 100, async_inflight_window => WindowSize, worker_pool_size => 1, resume_interval => 300 @@ -1157,7 +1162,8 @@ t_unblock_only_required_buffer_workers(_) -> #{name => test_resource}, #{ query_mode => async, - batch_size => 5 + batch_size => 5, + batch_time => 100 } ), lists:foreach( @@ -1171,7 +1177,8 @@ t_unblock_only_required_buffer_workers(_) -> #{name => test_resource}, #{ query_mode => async, - batch_size => 5 + batch_size => 5, + batch_time => 100 } ), %% creation of `?ID1` should not have unblocked `?ID`'s buffer workers @@ -1202,6 +1209,7 @@ t_retry_batch(_Config) -> #{ query_mode => async, batch_size => 5, + batch_time => 100, worker_pool_size => 1, resume_interval => 1_000 } @@ -1571,7 +1579,6 @@ t_retry_async_inflight_full(_Config) -> query_mode => async, async_inflight_window => AsyncInflightWindow, batch_size => 1, - batch_time => 20, worker_pool_size => 1, resume_interval => ResumeInterval } @@ -2086,7 +2093,6 @@ t_expiration_async_after_reply(_Config) -> #{ query_mode => async, batch_size => 1, - batch_time => 100, worker_pool_size => 1, resume_interval => 1_000 } @@ -2309,7 +2315,6 @@ t_expiration_retry(_Config) -> #{ query_mode => sync, batch_size => 1, - batch_time => 100, worker_pool_size => 1, resume_interval => 300 } @@ -2499,7 +2504,6 @@ t_recursive_flush(_Config) -> #{ query_mode => async, batch_size => 1, - batch_time => 10_000, worker_pool_size => 1 } ), From a9bc8a4464f42c092446a8c7d7a20cf5db0c5ab5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 14 Mar 2023 14:01:16 +0300 Subject: [PATCH 33/88] =?UTF-8?q?refactor(resman):=20rename=20`ets=5Flooku?= =?UTF-8?q?p`=20=E2=86=92=20`lookup=5Fcached`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That way we hide the impementation details + the interface becomes cleaner and more obvious. --- apps/emqx_resource/src/emqx_resource.erl | 4 ++-- .../src/emqx_resource_buffer_worker.erl | 2 +- apps/emqx_resource/src/emqx_resource_manager.erl | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 57d56b339..1ccb5ca71 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -260,7 +260,7 @@ query(ResId, Request) -> -spec query(resource_id(), Request :: term(), query_opts()) -> Result :: term(). query(ResId, Request, Opts) -> - case emqx_resource_manager:ets_lookup(ResId) of + case emqx_resource_manager:lookup_cached(ResId) of {ok, _Group, #{query_mode := QM, mod := Module}} -> IsBufferSupported = is_buffer_supported(Module), case {IsBufferSupported, QM} of @@ -311,7 +311,7 @@ set_resource_status_connecting(ResId) -> -spec get_instance(resource_id()) -> {ok, resource_group(), resource_data()} | {error, Reason :: term()}. get_instance(ResId) -> - emqx_resource_manager:ets_lookup(ResId, [metrics]). + emqx_resource_manager:lookup_cached(ResId, [metrics]). -spec fetch_creation_opts(map()) -> creation_opts(). fetch_creation_opts(Opts) -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 711833963..8bfd77e61 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -885,7 +885,7 @@ handle_async_worker_down(Data0, Pid) -> call_query(QM0, Id, Index, Ref, Query, QueryOpts) -> ?tp(call_query_enter, #{id => Id, query => Query, query_mode => QM0}), - case emqx_resource_manager:ets_lookup(Id) of + case emqx_resource_manager:lookup_cached(Id) of {ok, _Group, #{status := stopped}} -> ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); {ok, _Group, Resource} -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 0983dff8d..40f9fe1ab 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -36,8 +36,8 @@ lookup/1, list_all/0, list_group/1, - ets_lookup/1, - ets_lookup/2, + lookup_cached/1, + lookup_cached/2, get_metrics/1, reset_metrics/1 ]). @@ -231,21 +231,21 @@ set_resource_status_connecting(ResId) -> -spec lookup(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. lookup(ResId) -> case safe_call(ResId, lookup, ?T_LOOKUP) of - {error, timeout} -> ets_lookup(ResId, [metrics]); + {error, timeout} -> lookup_cached(ResId, [metrics]); Result -> Result end. %% @doc Lookup the group and data of a resource from the cache --spec ets_lookup(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. -ets_lookup(ResId) -> - ets_lookup(ResId, []). +-spec lookup_cached(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. +lookup_cached(ResId) -> + lookup_cached(ResId, []). %% @doc Lookup the group and data of a resource from the cache --spec ets_lookup(resource_id(), [Option]) -> +-spec lookup_cached(resource_id(), [Option]) -> {ok, resource_group(), resource_data()} | {error, not_found} when Option :: metrics. -ets_lookup(ResId, Options) -> +lookup_cached(ResId, Options) -> NeedMetrics = lists:member(metrics, Options), case read_cache(ResId) of {Group, Data} when NeedMetrics -> From 03b95073fc44b0a946f732e2802ac6930d76630e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Mar 2023 14:25:41 -0300 Subject: [PATCH 34/88] test: fix inter-suite flakiness --- apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index 4c349c7a0..61df9bd29 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -42,6 +42,8 @@ init_per_suite(_Config) -> []. end_per_suite(_Config) -> + ok = emqx_config:put([bridges], #{}), + ok = emqx_config:put_raw([bridges], #{}), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_bridge]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), _ = application:stop(emqx_connector), From 164440fe83ad71932544fe7dc5ebbbc2590d5ced Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Mar 2023 13:57:31 -0300 Subject: [PATCH 35/88] test(resource): fix flaky test Sometimes this test might retry more times, so we check the prefix of the trace only. --- apps/emqx_resource/test/emqx_resource_SUITE.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index c8b5ff183..b8593c377 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1167,10 +1167,11 @@ t_retry_batch(_Config) -> %% each time should be the original batch (no duplicate %% elements or reordering). ExpectedSeenPayloads = lists:flatten(lists:duplicate(4, Payloads)), - ?assertEqual( - ExpectedSeenPayloads, - ?projection(n, ?of_kind(connector_demo_batch_inc_individual, Trace)) + Trace1 = lists:sublist( + ?projection(n, ?of_kind(connector_demo_batch_inc_individual, Trace)), + length(ExpectedSeenPayloads) ), + ?assertEqual(ExpectedSeenPayloads, Trace1), ?assertMatch( [#{n := ExpectedCount}], ?of_kind(connector_demo_inc_counter, Trace) From d13d54fb815adcc6c551e3e2215b51dfdb157b87 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 19:08:51 +0100 Subject: [PATCH 36/88] ci: emqx-builder 5.0-32 --- .github/workflows/build_and_push_docker_images.yaml | 4 ++-- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 6 +++--- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 6 +++--- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 57dc2cb45..adf2c2b84 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -121,7 +121,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-29 # update to latest + - 5.0-32 # update to latest otp: - 24.3.4.2-2 # switch to 25 once ready to release 5.1 elixir: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fe65ad455..3141b77d5 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -24,7 +24,7 @@ jobs: prepare: runs-on: ubuntu-22.04 if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule' - container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -221,7 +221,7 @@ jobs: - aws-arm64 - ubuntu-22.04 builder: - - 5.0-29 + - 5.0-32 elixir: - 1.13.4 exclude: @@ -235,7 +235,7 @@ jobs: arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-29 + builder: 5.0-32 elixir: 1.13.4 release_with: elixir - profile: emqx @@ -243,7 +243,7 @@ jobs: arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-29 + builder: 5.0-32 elixir: 1.13.4 release_with: elixir diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index e18a1319e..08ed8ed2d 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -35,7 +35,7 @@ jobs: - ["emqx-enterprise", "24.3.4.2-2", "amzn2", "erlang"] - ["emqx-enterprise", "25.1.2-2", "ubuntu20.04", "erlang"] builder: - - 5.0-29 + - 5.0-32 elixir: - '1.13.4' diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index f42ada7ac..58dd06e30 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -6,7 +6,7 @@ on: jobs: check_deps_integrity: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 390ca8ffe..de05f7e59 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 208911eb1..181e81305 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -9,7 +9,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-22.04 # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index e990f69dc..d753693cc 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -8,7 +8,7 @@ on: jobs: elixir_deps_check: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index eb25d57bd..1647071af 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-25.1.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 147708373..52ba13373 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: builder: - - 5.0-29 + - 5.0-32 otp: - 24.3.4.2-2 - 25.1.2-2 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index edb582741..f729c8cbd 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-debian11 steps: - uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-29 + - 5.0-32 otp: - 24.3.4.2-2 elixir: @@ -123,7 +123,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-29 + - 5.0-32 otp: - 24.3.4.2-2 elixir: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 1e14eb2b3..cd969045d 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index e76b05f7e..c289fba5e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -31,12 +31,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-29", + builder: "5.0-32", otp: "25.1.2-2", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-29", + builder: "5.0-32", otp: ["24.3.4.2-2", "25.1.2-2"][], elixir: "1.13.4" }) @@ -225,7 +225,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-29:1.13.4-24.3.4.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 From 5258b4c6e99a43ab7e72c9eed267e71579c3db33 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 19:13:13 +0100 Subject: [PATCH 37/88] chore: bump ekka to 0.14.5 to make use of erlang-rocksdb 1.7.2-emqx-9 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 949a7a734..4b2c2a974 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.4"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/mix.exs b/mix.exs index 12dbd09dc..1befd534a 100644 --- a/mix.exs +++ b/mix.exs @@ -54,7 +54,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.14.4", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.14.5", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, diff --git a/rebar.config b/rebar.config index c3296518d..3eca8fcae 100644 --- a/rebar.config +++ b/rebar.config @@ -56,7 +56,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.4"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} From ac65ce294742a28469716ae94dc1d3cd06434892 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 19:33:08 +0100 Subject: [PATCH 38/88] chore: run xref before dialyzer in make static_checks --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 998175eea..370c861d6 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ ct: $(REBAR) merge-config ## only check bpapi for enterprise profile because it's a super-set. .PHONY: static_checks static_checks: - @$(REBAR) as check do dialyzer, xref + @$(REBAR) as check do xref, dialyzer @if [ "$${PROFILE}" = 'emqx-enterprise' ]; then $(REBAR) ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE); fi @if [ "$${PROFILE}" = 'emqx-enterprise' ]; then ./scripts/check-i18n-style.sh; fi From f21d01253221bc383f24a8fe9de87e215a08d51d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 19:39:08 +0100 Subject: [PATCH 39/88] ci: cache dialyzer plt --- .github/workflows/run_test_cases.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index c289fba5e..1efe7a4e7 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -69,6 +69,11 @@ jobs: - uses: actions/checkout@v3 with: path: source + - uses: actions/cache@v3 + id: cache + with: + path: "$HOME/.cache/rebar3/rebar3_${{ matrix.otp }}_plt" + key: rebar3-dialyzer-plt-${{ matrix.otp }} - name: get_all_deps working-directory: source env: From 19fb3854a8c66bf801621a59576a450c7bde80fe Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 15 Mar 2023 20:05:18 +0100 Subject: [PATCH 40/88] ci: run gitlint in docker container --- .github/workflows/run_gitlint.yaml | 38 ++++-------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml index b89d0b7b0..52082c56e 100644 --- a/.github/workflows/run_gitlint.yaml +++ b/.github/workflows/run_gitlint.yaml @@ -6,39 +6,11 @@ jobs: run_gitlint: runs-on: ubuntu-22.04 steps: - - name: Checkout source code - uses: actions/checkout@v3 - - name: Install gitlint - run: | - sudo apt-get update - sudo apt install gitlint - - name: Set auth header - if: endsWith(github.repository, 'enterprise') - run: | - echo 'AUTH_HEADER<> $GITHUB_ENV - echo "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Run gitlint shell: bash run: | - pr_number=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') - messages="$(curl --silent --show-error \ - --header "${{ env.AUTH_HEADER }}" \ - --header "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/commits")" - len=$(echo $messages | jq length) - result=true - for i in $( seq 0 $(($len - 1)) ); do - message=$(echo $messages | jq -r .[$i].commit.message) - echo "commit message: $message" - status=0 - echo $message | gitlint -C ./.github/workflows/.gitlint || status=$? - if [ $status -ne 0 ]; then - result=false - fi - done - if ! ${result} ; then - echo "Some of the commit messages are not structured as The Conventional Commits specification. Please check CONTRIBUTING.md for our process on PR." - exit 1 - fi - echo "success" + set -ex + docker run --ulimit nofile=1024 -v $(pwd):/repo -w /repo ghcr.io/emqx/gitlint --commits ${{ github.event.pull_request.base.sha }}..$GITHUB_SHA --config .github/workflows/.gitlint From abbe5be3eb4a86fd0ad5f13f645c813ee544dc80 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 16 Mar 2023 09:24:50 +0100 Subject: [PATCH 41/88] chore: use single default pr template --- .github/PULL_REQUEST_TEMPLATE/ci.md | 7 ------- .github/PULL_REQUEST_TEMPLATE/doc.md | 1 - .github/PULL_REQUEST_TEMPLATE/v4.md | 12 ------------ .../v5.md => pull_request_template.md} | 5 +++++ 4 files changed, 5 insertions(+), 20 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE/ci.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE/doc.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE/v4.md rename .github/{PULL_REQUEST_TEMPLATE/v5.md => pull_request_template.md} (68%) diff --git a/.github/PULL_REQUEST_TEMPLATE/ci.md b/.github/PULL_REQUEST_TEMPLATE/ci.md deleted file mode 100644 index 764933516..000000000 --- a/.github/PULL_REQUEST_TEMPLATE/ci.md +++ /dev/null @@ -1,7 +0,0 @@ -Fixes - -## PR Checklist -Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: - -- [ ] If changed package build ci, pass [this action](https://github.com/emqx/emqx/actions/workflows/build_packages.yaml) (manual trigger) -- [ ] Change log has been added to `changes/` dir for user-facing artifacts update diff --git a/.github/PULL_REQUEST_TEMPLATE/doc.md b/.github/PULL_REQUEST_TEMPLATE/doc.md deleted file mode 100644 index af1c9127f..000000000 --- a/.github/PULL_REQUEST_TEMPLATE/doc.md +++ /dev/null @@ -1 +0,0 @@ -Fixes diff --git a/.github/PULL_REQUEST_TEMPLATE/v4.md b/.github/PULL_REQUEST_TEMPLATE/v4.md deleted file mode 100644 index 11b282091..000000000 --- a/.github/PULL_REQUEST_TEMPLATE/v4.md +++ /dev/null @@ -1,12 +0,0 @@ -Fixes - -## PR Checklist -Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: - -- [ ] Added tests for the changes -- [ ] Changed lines covered in coverage report -- [ ] Change log has been added to `changes/` dir -- [ ] `appup` files updated (execute `scripts/update-appup.sh emqx`) -- [ ] For internal contributor: there is a jira ticket to track this change -- [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up -- [ ] In case of non-backward compatible changes, reviewer should check this item as a write-off, and add details in **Backward Compatibility** section diff --git a/.github/PULL_REQUEST_TEMPLATE/v5.md b/.github/pull_request_template.md similarity index 68% rename from .github/PULL_REQUEST_TEMPLATE/v5.md rename to .github/pull_request_template.md index 7952c1371..9b96db554 100644 --- a/.github/PULL_REQUEST_TEMPLATE/v5.md +++ b/.github/pull_request_template.md @@ -9,3 +9,8 @@ Please convert it to a draft if any of the following conditions are not met. Rev - [ ] For internal contributor: there is a jira ticket to track this change - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] Schema changes are backward compatible + +## Checklist for CI (.github/workflows) changes + +- [ ] If changed package build workflow, pass [this action](https://github.com/emqx/emqx/actions/workflows/build_packages.yaml) (manual trigger) +- [ ] Change log has been added to `changes/` dir for user-facing artifacts update From cfae0baf030008b60769e859bc918333fa9fa76e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 16 Mar 2023 09:32:13 +0100 Subject: [PATCH 42/88] ci: do not post updates about enterprise releases to emqx.io --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 31afba81d..2f5ddf171 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,9 +69,9 @@ jobs: with: asset_paths: '["packages/*"]' - name: update to emqx.io - if: github.event_name == 'release' || inputs.publish_release_artefacts + if: startsWith(github.ref_name, 'v') && (github.event_name == 'release' || inputs.publish_release_artefacts) run: | - set -e -x -u + set -eux curl -w %{http_code} \ --insecure \ -H "Content-Type: application/json" \ From d147299e23f47f5d4914e88568f95b164a439aa9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 16 Mar 2023 17:35:06 +0800 Subject: [PATCH 43/88] chore: refine changes for merged PRs --- changes/ce/feat-10065.en.md | 1 + changes/ce/feat-10065.zh.md | 1 + changes/ce/feat-9893.en.md | 2 ++ changes/ce/feat-9893.zh.md | 2 ++ changes/ce/fix-10056.en.md | 4 +++- changes/ce/fix-10056.zh.md | 4 +++- changes/ce/fix-10066.en.md | 2 +- changes/ce/fix-10066.zh.md | 2 +- changes/ce/fix-10086.en.md | 2 +- changes/ce/fix-10086.zh.md | 2 +- changes/ce/fix-10100.en.md | 1 + changes/ce/fix-10100.zh.md | 1 + changes/ce/fix-10118.en.md | 2 +- changes/ce/fix-10118.zh.md | 4 ++-- changes/ee/feat-10083.en.md | 2 +- changes/ee/feat-10083.zh.md | 2 +- changes/v5.0.16/feat-9893.en.md | 1 - changes/v5.0.16/feat-9893.zh.md | 1 - 18 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 changes/ce/feat-10065.en.md create mode 100644 changes/ce/feat-10065.zh.md create mode 100644 changes/ce/feat-9893.en.md create mode 100644 changes/ce/feat-9893.zh.md delete mode 100644 changes/v5.0.16/feat-9893.en.md delete mode 100644 changes/v5.0.16/feat-9893.zh.md diff --git a/changes/ce/feat-10065.en.md b/changes/ce/feat-10065.en.md new file mode 100644 index 000000000..ae182f3c8 --- /dev/null +++ b/changes/ce/feat-10065.en.md @@ -0,0 +1 @@ +Add deb package support for `raspbian9` and `raspbian10`. diff --git a/changes/ce/feat-10065.zh.md b/changes/ce/feat-10065.zh.md new file mode 100644 index 000000000..366276333 --- /dev/null +++ b/changes/ce/feat-10065.zh.md @@ -0,0 +1 @@ +为 `raspbian9` 及 `raspbian10` 增加 deb 包支持。 diff --git a/changes/ce/feat-9893.en.md b/changes/ce/feat-9893.en.md new file mode 100644 index 000000000..343c3794f --- /dev/null +++ b/changes/ce/feat-9893.en.md @@ -0,0 +1,2 @@ +When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by banned clients. +Previously, the messages sent by banned clients may still be delivered to subscribers in this scenario. diff --git a/changes/ce/feat-9893.zh.md b/changes/ce/feat-9893.zh.md new file mode 100644 index 000000000..426439c3e --- /dev/null +++ b/changes/ce/feat-9893.zh.md @@ -0,0 +1,2 @@ +当使用 `clean_start=false` 标志连接时，EMQX 将会从消息队列中过滤出被封禁客户端发出的消息，使它们不能被下发给订阅者。 +此前被封禁客户端发出的消息仍可能在这一场景下被下发给订阅者。 diff --git a/changes/ce/fix-10056.en.md b/changes/ce/fix-10056.en.md index ab9b980e8..55449294d 100644 --- a/changes/ce/fix-10056.en.md +++ b/changes/ce/fix-10056.en.md @@ -1 +1,3 @@ -`/bridges` API: return `400` instead of `403` in case of inconsistency in the application logic either because bridge is about to be deleted, but active rules still depend on it, or an operation (start|stop|restart) is called, but the bridge is not enabled. +Fix `/bridges` API status code. +- Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule. +- Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled. diff --git a/changes/ce/fix-10056.zh.md b/changes/ce/fix-10056.zh.md index 4d3317165..ec5982137 100644 --- a/changes/ce/fix-10056.zh.md +++ b/changes/ce/fix-10056.zh.md @@ -1 +1,3 @@ -`/bridges` API：在应用逻辑不一致的情况下，返回`400'而不是`403'，因为桥即将被删除，但活动规则仍然依赖于它，或者调用了一个操作（启动|停止|重新启动），但桥没有被启用。 +修复 `/bridges` API 的 HTTP 状态码。 +- 当删除被活动中的规则依赖的数据桥接时，将返回 `400` 而不是 `403` 。 +- 当数据桥接未启用时，调用操作（启动|停止|重启）将返回 `400` 而不是 `403`。 diff --git a/changes/ce/fix-10066.en.md b/changes/ce/fix-10066.en.md index 2d23ad5b9..87e253aca 100644 --- a/changes/ce/fix-10066.en.md +++ b/changes/ce/fix-10066.en.md @@ -1 +1 @@ -Return human readable error message for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls and set HTTP status code to `400` instead of `500`. +Improve error messages for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls to make them more readable. And set HTTP status code to `400` instead of `500`. diff --git a/changes/ce/fix-10066.zh.md b/changes/ce/fix-10066.zh.md index c72f21ff1..e5e3c2113 100644 --- a/changes/ce/fix-10066.zh.md +++ b/changes/ce/fix-10066.zh.md @@ -1 +1 @@ -为 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` 的 API 调用返回人类可读的错误信息，并将 HTTP 状态代码设置为 `400` 而不是 `500`。 +改进 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误信息，使之更加易读。并将 HTTP 状态代码设置为 `400` 而不是 `500`。 diff --git a/changes/ce/fix-10086.en.md b/changes/ce/fix-10086.en.md index 31e8b6453..d337a57c7 100644 --- a/changes/ce/fix-10086.en.md +++ b/changes/ce/fix-10086.en.md @@ -1,4 +1,4 @@ Upgrade HTTP client ehttpc to `0.4.7`. Prior to this upgrade, HTTP clients for authentication, authorization and webhook may crash -if `body` is empty but content-type HTTP header is set. +if `Body` is empty but `Content-Type` HTTP header is set. For more details see [ehttpc PR#44](https://github.com/emqx/ehttpc/pull/44). diff --git a/changes/ce/fix-10086.zh.md b/changes/ce/fix-10086.zh.md index b7c110ea4..c083d6055 100644 --- a/changes/ce/fix-10086.zh.md +++ b/changes/ce/fix-10086.zh.md @@ -1,3 +1,3 @@ HTTP 客户端库 `ehttpc` 升级到 0.4.7。 -在升级前，如果 HTTP 客户端，例如 认证，授权，webhook 等配置中使用了content-type HTTP 头，但是没有配置 body，则可能会发生异常。 +在升级前，如果 HTTP 客户端，例如 '认证'、'授权'、'WebHook' 等配置中使用了 `Content-Type` HTTP 头，但是没有配置 `Body`，则可能会发生异常。 详情见 [ehttpc PR#44](https://github.com/emqx/ehttpc/pull/44)。 diff --git a/changes/ce/fix-10100.en.md b/changes/ce/fix-10100.en.md index 002fb6f08..e16ee5efc 100644 --- a/changes/ce/fix-10100.en.md +++ b/changes/ce/fix-10100.en.md @@ -1 +1,2 @@ Fix channel crash for slow clients with enhanced authentication. +Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash. diff --git a/changes/ce/fix-10100.zh.md b/changes/ce/fix-10100.zh.md index 6adb5e7e1..ac2483a27 100644 --- a/changes/ce/fix-10100.zh.md +++ b/changes/ce/fix-10100.zh.md @@ -1 +1,2 @@ 修复响应较慢的客户端在使用增强认证时可能出现崩溃的问题。 +此前，当客户端使用增强认证功能，但发送 Auth 报文较慢或 Auth 报文丢失时会导致客户端进程崩溃。 diff --git a/changes/ce/fix-10118.en.md b/changes/ce/fix-10118.en.md index dd6b5129f..f6db758f3 100644 --- a/changes/ce/fix-10118.en.md +++ b/changes/ce/fix-10118.en.md @@ -1,4 +1,4 @@ Fix problems related to manual joining of EMQX replicant nodes to the cluster. -Previously, manually joining and then leaving the cluster rendered replicant node unable to start EMQX again and required a node restart. +Previously, after manually executing joining and then leaving the cluster, the `replicant` node can only run normally after restarting the node after joining the cluster again. [Mria PR](https://github.com/emqx/mria/pull/128) diff --git a/changes/ce/fix-10118.zh.md b/changes/ce/fix-10118.zh.md index 4334a5bba..a037215f0 100644 --- a/changes/ce/fix-10118.zh.md +++ b/changes/ce/fix-10118.zh.md @@ -1,4 +1,4 @@ -修复与手动加入 EMQX `replicant` 节点到集群有关的问题。 -以前，手动加入然后离开集群会使 `replicant` 节点无法再次启动 EMQX，需要重新启动节点。 +修复 `replicant` 节点因为手动加入 EMQX 集群导致的相关问题。 +此前，手动执行 `加入集群-离开集群` 后，`replicant` 节点再次加入集群后只有重启节点才能正常运行。 [Mria PR](https://github.com/emqx/mria/pull/128) diff --git a/changes/ee/feat-10083.en.md b/changes/ee/feat-10083.en.md index 635549d5e..f4331faf9 100644 --- a/changes/ee/feat-10083.en.md +++ b/changes/ee/feat-10083.en.md @@ -1 +1 @@ -Integrate `DynamoDB` into `bridges` as a new backend. +Add `DynamoDB` support for Data-Brdige. diff --git a/changes/ee/feat-10083.zh.md b/changes/ee/feat-10083.zh.md index 061e2e416..8274e62c2 100644 --- a/changes/ee/feat-10083.zh.md +++ b/changes/ee/feat-10083.zh.md @@ -1 +1 @@ -在 `桥接` 中集成 `DynamoDB`。 +为数据桥接增加 `DynamoDB` 支持。 diff --git a/changes/v5.0.16/feat-9893.en.md b/changes/v5.0.16/feat-9893.en.md deleted file mode 100644 index 590d82a0f..000000000 --- a/changes/v5.0.16/feat-9893.en.md +++ /dev/null @@ -1 +0,0 @@ -When connecting with the flag `clean_start=false`, the new session will filter out banned messages from the `mqueue` before deliver. diff --git a/changes/v5.0.16/feat-9893.zh.md b/changes/v5.0.16/feat-9893.zh.md deleted file mode 100644 index 30286a679..000000000 --- a/changes/v5.0.16/feat-9893.zh.md +++ /dev/null @@ -1 +0,0 @@ -当使用 `clean_start=false` 标志连接时，新会话将在传递之前从 `mqueue` 中过滤掉被封禁的消息。 From afd29b69c48ba8725321c54c01b7d5f59104d952 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 16 Mar 2023 17:50:37 +0800 Subject: [PATCH 44/88] chore: refine i18n doc punctuation --- apps/emqx/i18n/emqx_schema_i18n.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index b57698327..fb458b449 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1079,11 +1079,11 @@ Supported configurations are the following: zh: """共享订阅消息派发策略。 - `random`：随机挑选一个共享订阅者派发； - `round_robin`：使用 round-robin 策略派发； - - `round_robin_per_group`: 在共享组内循环选择下一个成员； - - `local`: 选择随机的本地成员，否则选择随机的集群范围内成员; + - `round_robin_per_group`：在共享组内循环选择下一个成员； + - `local`：选择随机的本地成员，否则选择随机的集群范围内成员; - `sticky`：总是使用上次选中的订阅者派发，直到它断开连接； - `hash_clientid`：使用发送者的 Client ID 进行 Hash 来选择订阅者; - - `hash_topic`: 使用源主题进行 Hash 来选择订阅者。""" + - `hash_topic`：使用源主题进行 Hash 来选择订阅者。""" } } @@ -1095,7 +1095,7 @@ This should allow messages to be dispatched to a different subscriber in the gro zh: """该配置项已废弃，会在 5.1 中移除。 启用/禁用 QoS 1 和 QoS 2 消息的共享派发确认。 -开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线）重新派发给另外一个订阅者。""" +开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线） 重新派发给另外一个订阅者。""" } } From e5645a7b21e7eaede15dc43ceb65fc069d061327 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 09:18:16 -0300 Subject: [PATCH 45/88] refactor: rename macros --- apps/emqx/src/emqx_tls_lib.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index eb8234547..47797b326 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -47,13 +47,13 @@ -define(IS_TRUE(Val), ((Val =:= true) orelse (Val =:= <<"true">>))). -define(IS_FALSE(Val), ((Val =:= false) orelse (Val =:= <<"false">>))). --define(SSL_FILE_OPT_NAMES, [ +-define(SSL_FILE_OPT_PATHS, [ [<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>], [<<"ocsp">>, <<"issuer_pem">>] ]). --define(SSL_FILE_OPT_NAMES_A, [ +-define(SSL_FILE_OPT_PATHS_A, [ [keyfile], [certfile], [cacertfile], @@ -308,7 +308,7 @@ ensure_ssl_files(Dir, SSL, Opts) -> RequiredKeys = maps:get(required_keys, Opts, []), case ensure_ssl_file_key(SSL, RequiredKeys) of ok -> - KeyPaths = ?SSL_FILE_OPT_NAMES ++ ?SSL_FILE_OPT_NAMES_A, + KeyPaths = ?SSL_FILE_OPT_PATHS ++ ?SSL_FILE_OPT_PATHS_A, ensure_ssl_files(Dir, SSL, KeyPaths, Opts); {error, _} = Error -> Error @@ -336,7 +336,7 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> end, lists:foreach( fun(KeyPath) -> delete_old_file(Get(KeyPath, NewOpts), Get(KeyPath, OldOpts)) end, - ?SSL_FILE_OPT_NAMES ++ ?SSL_FILE_OPT_NAMES_A + ?SSL_FILE_OPT_PATHS ++ ?SSL_FILE_OPT_PATHS_A ), %% try to delete the dir if it is empty _ = file:del_dir(pem_dir(Dir)), @@ -482,13 +482,13 @@ is_valid_pem_file(Path) -> %% so they are forced to upload a cert file, or use an existing file path. -spec drop_invalid_certs(map()) -> map(). drop_invalid_certs(#{enable := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_NAMES_A); + lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); drop_invalid_certs(#{<<"enable">> := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_NAMES); + lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); drop_invalid_certs(#{enable := True} = SSL) when ?IS_TRUE(True) -> - do_drop_invalid_certs(?SSL_FILE_OPT_NAMES_A, SSL); + do_drop_invalid_certs(?SSL_FILE_OPT_PATHS_A, SSL); drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> - do_drop_invalid_certs(?SSL_FILE_OPT_NAMES, SSL). + do_drop_invalid_certs(?SSL_FILE_OPT_PATHS, SSL). do_drop_invalid_certs([], SSL) -> SSL; From d1f58d6e2dd4e81c87accee156a92cb493781d85 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 09:32:40 -0300 Subject: [PATCH 46/88] refactor: replace macro by simple function --- apps/emqx/src/emqx_ocsp_cache.erl | 114 +++++++++++++++--------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 25c6200ae..17cd93d83 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -55,35 +55,6 @@ -define(MIN_REFRESH_INTERVAL, timer:minutes(1)). -endif. --define(WITH_LISTENER_CONFIG(ListenerID, ConfPath, Pattern, ErrorResp, Action), - case emqx_listeners:parse_listener_id(ListenerID) of - {ok, #{type := Type, name := Name}} -> - case emqx_config:get_listener_conf(Type, Name, ConfPath, not_found) of - not_found -> - ?SLOG(error, #{ - msg => "listener_config_missing", - listener_id => ListenerID - }), - (ErrorResp); - Pattern -> - Action; - OtherConfig -> - ?SLOG(error, #{ - msg => "listener_config_inconsistent", - listener_id => ListenerID, - config => OtherConfig - }), - (ErrorResp) - end; - _Err -> - ?SLOG(error, #{ - msg => "listener_id_not_found", - listener_id => ListenerID - }), - (ErrorResp) - end -). - %% Allow usage of OTP certificate record fields (camelCase). -elvis([ {elvis_style, atom_naming_convention, #{ @@ -231,22 +202,45 @@ http_fetch(ListenerID) -> %% TODO: configurable call timeout? gen_server:call(?MODULE, {http_fetch, ListenerID}, ?CALL_TIMEOUT). +with_listener_config(ListenerID, ConfPath, ErrorResp, Fn) -> + case emqx_listeners:parse_listener_id(ListenerID) of + {ok, #{type := Type, name := Name}} -> + case emqx_config:get_listener_conf(Type, Name, ConfPath, not_found) of + not_found -> + ?SLOG(error, #{ + msg => "listener_config_missing", + listener_id => ListenerID + }), + ErrorResp; + Config -> + Fn(Config) + end; + _Err -> + ?SLOG(error, #{ + msg => "listener_id_not_found", + listener_id => ListenerID + }), + ErrorResp + end. + cache_key(ListenerID) -> - ?WITH_LISTENER_CONFIG( - ListenerID, - [ssl_options], - #{certfile := ServerCertPemPath}, - error, - begin + with_listener_config(ListenerID, [ssl_options], error, fun + (#{certfile := ServerCertPemPath}) -> #'Certificate'{ tbsCertificate = #'TBSCertificate'{ signature = Signature } } = read_server_cert(ServerCertPemPath), - {ok, {ocsp_response, Signature}} - end - ). + {ok, {ocsp_response, Signature}}; + (OtherConfig) -> + ?SLOG(error, #{ + msg => "listener_config_inconsistent", + listener_id => ListenerID, + config => OtherConfig + }), + error + end). do_lookup(ListenerID) -> CacheKey = cache_key(ListenerID), @@ -311,25 +305,31 @@ with_refresh_params(ListenerID, Conf, ErrorRet, Fn) -> get_refresh_params(ListenerID, undefined = _Conf) -> %% during normal periodic refreshes, we read from the emqx config. - ?WITH_LISTENER_CONFIG( - ListenerID, - [ssl_options], - #{ - ocsp := #{ - issuer_pem := IssuerPemPath, - responder_url := ResponderURL, - refresh_http_timeout := HTTPTimeout - }, - certfile := ServerCertPemPath - }, - error, - {ok, #{ - issuer_pem => IssuerPemPath, - responder_url => ResponderURL, - refresh_http_timeout => HTTPTimeout, - server_certfile => ServerCertPemPath - }} - ); + with_listener_config(ListenerID, [ssl_options], error, fun + ( + #{ + ocsp := #{ + issuer_pem := IssuerPemPath, + responder_url := ResponderURL, + refresh_http_timeout := HTTPTimeout + }, + certfile := ServerCertPemPath + } + ) -> + {ok, #{ + issuer_pem => IssuerPemPath, + responder_url => ResponderURL, + refresh_http_timeout => HTTPTimeout, + server_certfile => ServerCertPemPath + }}; + (OtherConfig) -> + ?SLOG(error, #{ + msg => "listener_config_inconsistent", + listener_id => ListenerID, + config => OtherConfig + }), + error + end); get_refresh_params(_ListenerID, #{ ssl_options := #{ ocsp := #{ From a614bdc94a487af899a0f0a86fa123227b1e8595 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 09:55:52 -0300 Subject: [PATCH 47/88] chore(ocsp): catch unexpected error when fetching ocsp response --- apps/emqx/src/emqx_ocsp_cache.erl | 12 +++++++++++- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 17cd93d83..4e7ada044 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -300,7 +300,17 @@ with_refresh_params(ListenerID, Conf, ErrorRet, Fn) -> error -> ErrorRet; {ok, Params} -> - Fn(Params) + try + Fn(Params) + catch + Kind:Error -> + ?SLOG(error, #{ + msg => "error_fetching_ocsp_response", + listener_id => ListenerID, + error => {Kind, Error} + }), + ErrorRet + end end. get_refresh_params(ListenerID, undefined = _Conf) -> diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index e0c29e440..c45bc15ef 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -912,6 +912,24 @@ do_t_validations(_Config) -> ok. +t_unknown_error_fetching_ocsp_response(_Config) -> + ListenerID = <<"ssl:test_ocsp">>, + TestPid = self(), + ok = meck:expect( + emqx_ocsp_cache, + http_get, + fun(_RequestURI, _HTTPTimeout) -> + TestPid ! error_raised, + meck:exception(error, something_went_wrong) + end + ), + ?assertEqual(error, emqx_ocsp_cache:fetch_response(ListenerID)), + receive + error_raised -> ok + after 200 -> ct:fail("should have tried to fetch ocsp response") + end, + ok. + t_openssl_client(Config) -> TLSVsn = ?config(tls_vsn, Config), WithStatusRequest = ?config(status_request, Config), From 75dad647a94a8e7a2437b685899312bec0f1d699 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 09:57:01 -0300 Subject: [PATCH 48/88] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- changes/ce/feat-10128.zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10128.zh.md b/changes/ce/feat-10128.zh.md index 544ea400e..d875bd2ff 100644 --- a/changes/ce/feat-10128.zh.md +++ b/changes/ce/feat-10128.zh.md @@ -1 +1 @@ -为 SSL MQTT 监听器增加对 OCSP Stapling 和 CRL 检查的支持。 +为 SSL MQTT 监听器增加对 OCSP Stapling 的支持。 From 51a0b93868f6f12d84bcde3b4272ce82e002dfc9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Feb 2023 15:22:11 -0300 Subject: [PATCH 49/88] test: ensure configs are up to date before running suites --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 370c861d6..75c27d910 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ endef $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app)))) .PHONY: ct-suite -ct-suite: $(REBAR) +ct-suite: $(REBAR) merge-config ifneq ($(TESTCASE),) ifneq ($(GROUP),) $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) --group $(GROUP) From 5ab5236ad3d17d94c37a14bb7523d9ebf0d4b28a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Feb 2023 10:14:58 -0300 Subject: [PATCH 50/88] test: fix flaky test --- .../emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 452b7a4d2..8424ddff0 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -1105,13 +1105,13 @@ do_econnrefused_or_timeout_test(Config, Error) -> ?assertMatch( #{ dropped := Dropped, - failed := 0, + failed := Failed, inflight := Inflight, matched := Matched, queuing := Queueing, retried := 0, success := 0 - } when Matched >= 1 andalso Inflight + Queueing + Dropped =< 2, + } when Matched >= 1 andalso Inflight + Queueing + Dropped + Failed =< 2, CurrentMetrics ); {timeout, async} -> From 561c25f0e3f50d5a4431e3e506e447f2db14619d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 17 Feb 2023 15:03:20 -0300 Subject: [PATCH 51/88] feat: update snabbkaffe -> 1.0.7 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index b62ca6b3c..229979f6c 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -33,7 +33,7 @@ {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, - {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} + {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} ]}. {plugins, [{rebar3_proper, "0.12.1"}]}. diff --git a/mix.exs b/mix.exs index e946c257b..4f4174ee2 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do {:telemetry, "1.1.0"}, # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, - {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, + {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, {:hocon, github: "emqx/hocon", tag: "0.37.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, diff --git a/rebar.config b/rebar.config index 8bfcc7960..5ce9138ce 100644 --- a/rebar.config +++ b/rebar.config @@ -68,7 +68,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} - , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} + , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} From b9e92173cfc2dc6770d6be44557a674edd6005fb Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Feb 2023 14:08:33 -0300 Subject: [PATCH 52/88] test: improve cluster helper 1) Make each node have its own isolated data dir to avoid false negatives. 2) Allow parameterizing the peer module. 3) Fix cluster RPC after a node joins the cluster. --- apps/emqx/test/emqx_common_test_helpers.erl | 100 ++++++++++++++++---- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 4 +- 2 files changed, 85 insertions(+), 19 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index c26e63a62..8d2bd3ba3 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -37,6 +37,7 @@ deps_path/2, flush/0, flush/1, + load/1, render_and_load_app_config/1, render_and_load_app_config/2 ]). @@ -637,25 +638,53 @@ emqx_cluster(Specs0, CommonOpts) -> %% Lower level starting API -spec start_slave(shortname(), node_opts()) -> nodename(). -start_slave(Name, Opts) -> - {ok, Node} = ct_slave:start( - list_to_atom(atom_to_list(Name) ++ "@" ++ host()), - [ - {kill_if_fail, true}, - {monitor_master, true}, - {init_timeout, 10000}, - {startup_timeout, 10000}, - {erl_flags, erl_flags()} - ] - ), - +start_slave(Name, Opts) when is_list(Opts) -> + start_slave(Name, maps:from_list(Opts)); +start_slave(Name, Opts) when is_map(Opts) -> + SlaveMod = maps:get(peer_mod, Opts, ct_slave), + Node = node_name(Name), + DoStart = + fun() -> + case SlaveMod of + ct_slave -> + ct_slave:start( + Node, + [ + {kill_if_fail, true}, + {monitor_master, true}, + {init_timeout, 10000}, + {startup_timeout, 10000}, + {erl_flags, erl_flags()} + ] + ); + slave -> + slave:start_link(host(), Name, ebin_path()) + end + end, + case DoStart() of + {ok, _} -> + ok; + {error, started_not_connected, _} -> + ok; + Other -> + throw(Other) + end, pong = net_adm:ping(Node), + put_peer_mod(Node, SlaveMod), setup_node(Node, Opts), + ok = snabbkaffe:forward_trace(Node), Node. %% Node stopping -stop_slave(Node) -> - ct_slave:stop(Node). +stop_slave(Node0) -> + Node = node_name(Node0), + SlaveMod = get_peer_mod(Node), + erase_peer_mod(Node), + case SlaveMod:stop(Node) of + ok -> ok; + {ok, _} -> ok; + {error, not_started, _} -> ok + end. %% EPMD starting start_epmd() -> @@ -693,9 +722,12 @@ setup_node(Node, Opts) when is_map(Opts) -> {Type, listener_port(BasePort, Type)} || Type <- [tcp, ssl, ws, wss] ]), + %% we need a fresh data dir for each peer node to avoid unintended + %% successes due to sharing of data in the cluster. + PrivDataDir = maps:get(priv_data_dir, Opts, "/tmp"), %% Load env before doing anything to avoid overriding - [ok = rpc:call(Node, application, load, [App]) || App <- LoadApps], + lists:foreach(fun(App) -> rpc:call(Node, ?MODULE, load, [App]) end, LoadApps), %% Needs to be set explicitly because ekka:start() (which calls `gen`) is called without Handler %% in emqx_common_test_helpers:start_apps(...) @@ -721,7 +753,19 @@ setup_node(Node, Opts) when is_map(Opts) -> %% Otherwise, configuration gets loaded and all preset env in EnvHandler is lost LoadSchema andalso begin + %% to avoid sharing data between executions and/or + %% nodes. these variables might notbe in the + %% config file (e.g.: emqx_ee_conf_schema). + NodeDataDir = filename:join([ + PrivDataDir, + node(), + integer_to_list(erlang:unique_integer()) + ]), + os:putenv("EMQX_NODE__DATA_DIR", NodeDataDir), + os:putenv("EMQX_NODE__COOKIE", atom_to_list(erlang:get_cookie())), emqx_config:init_load(SchemaMod), + os:unsetenv("EMQX_NODE__DATA_DIR"), + os:unsetenv("EMQX_NODE__COOKIE"), application:set_env(emqx, init_config_load_done, true) end, @@ -750,6 +794,11 @@ setup_node(Node, Opts) when is_map(Opts) -> _ -> case rpc:call(Node, ekka, join, [JoinTo]) of ok -> + %% fix cluster rpc, as the conf app is not + %% restarted with the current test procedure. + StartApps andalso + lists:member(emqx_conf, Apps) andalso + (ok = erpc:call(Node, emqx_cluster_rpc, reset, [])), ok; ignore -> ok; @@ -762,8 +811,27 @@ setup_node(Node, Opts) when is_map(Opts) -> %% Helpers +put_peer_mod(Node, SlaveMod) -> + put({?MODULE, Node}, SlaveMod), + ok. + +get_peer_mod(Node) -> + case get({?MODULE, Node}) of + undefined -> ct_slave; + SlaveMod -> SlaveMod + end. + +erase_peer_mod(Node) -> + erase({?MODULE, Node}). + node_name(Name) -> - list_to_atom(lists:concat([Name, "@", host()])). + case string:tokens(atom_to_list(Name), "@") of + [_Name, _Host] -> + %% the name already has a @ + Name; + _ -> + list_to_atom(atom_to_list(Name) ++ "@" ++ host()) + end. gen_node_name(Num) -> list_to_atom("autocluster_node" ++ integer_to_list(Num)). diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 84ced5362..c34eb9dc3 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -25,7 +25,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_copy_conf_override_on_restarts(_Config) -> - net_kernel:start(['master@127.0.0.1', longnames]), ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), Cluster = cluster([core, core, core]), @@ -165,11 +164,10 @@ cluster(Specs) -> {env, Env}, {apps, [emqx_conf]}, {load_schema, false}, - {join_to, false}, + {join_to, true}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, []), - io:format("~p~p~n", [node(), application:get_all_env(emqx)]), ok; (_) -> ok From 1051df7af801aacc35e4fad8cd7fe836cc126174 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Feb 2023 14:10:58 -0300 Subject: [PATCH 53/88] test: add kafka to toxiproxy --- .../docker-compose-kafka.yaml | 26 +++++++++++-------- .../docker-compose-toxiproxy.yaml | 5 +++- .ci/docker-compose-file/docker-compose.yaml | 5 +--- .ci/docker-compose-file/toxiproxy.json | 24 +++++++++++++++++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 63e74fa11..e54f1377d 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -13,7 +13,7 @@ services: image: fredrikhgrelland/alpine-jdk11-openssl container_name: ssl_cert_gen volumes: - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kafka/generate-certs.sh:/bin/generate-certs.sh entrypoint: /bin/sh command: /bin/generate-certs.sh @@ -21,21 +21,24 @@ services: hostname: kdc.emqx.net image: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04 container_name: kdc.emqx.net + expose: + - 88 # kdc + - 749 # admin server + # ports: + # - 88:88 + # - 749:749 networks: emqx_bridge: volumes: - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf - ./kerberos/run.sh:/usr/bin/run.sh command: run.sh kafka_1: image: wurstmeister/kafka:2.13-2.7.0 - ports: - - "9092:9092" - - "9093:9093" - - "9094:9094" - - "9095:9095" + # ports: + # - "9192-9195:9192-9195" container_name: kafka-1.emqx.net hostname: kafka-1.emqx.net depends_on: @@ -48,9 +51,9 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093,SSL://:9094,SASL_SSL://:9095 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093,SSL://kafka-1.emqx.net:9094,SASL_SSL://kafka-1.emqx.net:9095 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL + KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093,SSL://:9094,SASL_SSL://:9095,LOCAL_PLAINTEXT://:9192,LOCAL_SASL_PLAINTEXT://:9193,LOCAL_SSL://:9194,LOCAL_SASL_SSL://:9195,TOXIPROXY_PLAINTEXT://:9292,TOXIPROXY_SASL_PLAINTEXT://:9293,TOXIPROXY_SSL://:9294,TOXIPROXY_SASL_SSL://:9295 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093,SSL://kafka-1.emqx.net:9094,SASL_SSL://kafka-1.emqx.net:9095,LOCAL_PLAINTEXT://localhost:9192,LOCAL_SASL_PLAINTEXT://localhost:9193,LOCAL_SSL://localhost:9194,LOCAL_SASL_SSL://localhost:9195,TOXIPROXY_PLAINTEXT://toxiproxy.emqx.net:9292,TOXIPROXY_SASL_PLAINTEXT://toxiproxy.emqx.net:9293,TOXIPROXY_SSL://toxiproxy.emqx.net:9294,TOXIPROXY_SASL_SSL://toxiproxy.emqx.net:9295 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL,LOCAL_PLAINTEXT:PLAINTEXT,LOCAL_SASL_PLAINTEXT:SASL_PLAINTEXT,LOCAL_SSL:SSL,LOCAL_SASL_SSL:SASL_SSL,TOXIPROXY_PLAINTEXT:PLAINTEXT,TOXIPROXY_SASL_PLAINTEXT:SASL_PLAINTEXT,TOXIPROXY_SSL:SSL,TOXIPROXY_SASL_SSL:SASL_SSL KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka @@ -58,6 +61,7 @@ services: KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks KAFKA_SSL_TRUSTSTORE_PASSWORD: password @@ -67,7 +71,7 @@ services: networks: emqx_bridge: volumes: - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kafka/jaas.conf:/etc/kafka/jaas.conf - ./kafka/kafka-entrypoint.sh:/bin/kafka-entrypoint.sh - ./kerberos/krb5.conf:/etc/kdc/krb5.conf diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 3dd30af52..16f18b6c2 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -6,7 +6,10 @@ services: image: ghcr.io/shopify/toxiproxy:2.5.0 restart: always networks: - - emqx_bridge + emqx_bridge: + aliases: + - toxiproxy + - toxiproxy.emqx.net volumes: - "./toxiproxy.json:/config/toxiproxy.json" ports: diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index ff330872d..42003fcb7 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -18,7 +18,7 @@ services: - emqx_bridge volumes: - ../..:/emqx - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf working_dir: /emqx @@ -37,6 +37,3 @@ networks: gateway: 172.100.239.1 - subnet: 2001:3200:3200::/64 gateway: 2001:3200:3200::1 - -volumes: # add this section - emqx-shared-secret: # does not need anything underneath this diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 6188eab17..2f8c4341b 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -53,5 +53,29 @@ "listen": "0.0.0.0:8000", "upstream": "dynamo:8000", "enabled": true + }, + { + "name": "kafka_plain", + "listen": "0.0.0.0:9292", + "upstream": "kafka-1.emqx.net:9292", + "enabled": true + }, + { + "name": "kafka_sasl_plain", + "listen": "0.0.0.0:9293", + "upstream": "kafka-1.emqx.net:9293", + "enabled": true + }, + { + "name": "kafka_ssl", + "listen": "0.0.0.0:9294", + "upstream": "kafka-1.emqx.net:9294", + "enabled": true + }, + { + "name": "kafka_sasl_ssl", + "listen": "0.0.0.0:9295", + "upstream": "kafka-1.emqx.net:9295", + "enabled": true } ] From 094e4a2eeb5cb63054fcc5fcd384f0ae99197981 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Feb 2023 14:11:20 -0300 Subject: [PATCH 54/88] chore: fix typespec --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index b2f06f35a..89f678554 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -204,7 +204,7 @@ do_multicall(M, F, A, RequiredSyncs, Timeout) -> query(TnxId) -> transaction(fun ?MODULE:trans_query/1, [TnxId]). --spec reset() -> reset. +-spec reset() -> ok. reset() -> gen_server:call(?MODULE, reset). -spec status() -> {'atomic', [map()]} | {'aborted', Reason :: term()}. From 969fa03ecc7d067c15c73eefbef7dc33f5b524e1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Feb 2023 14:13:56 -0300 Subject: [PATCH 55/88] feat: implement kafka consumer --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.erl | 12 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 11 +- apps/emqx_bridge/src/emqx_bridge_sup.erl | 2 - changes/ee/feat-9564.en.md | 3 + changes/ee/feat-9564.zh.md | 2 + .../i18n/emqx_ee_bridge_kafka.conf | 146 +- lib-ee/emqx_ee_bridge/rebar.config | 2 +- .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 25 +- .../src/emqx_ee_bridge_kafka.erl | 74 +- .../src/kafka/emqx_bridge_impl_kafka.erl | 52 +- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 381 +++++ .../kafka/emqx_bridge_impl_kafka_producer.erl | 62 +- .../emqx_ee_bridge_kafka_consumer_sup.erl | 79 + .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 1351 +++++++++++++++++ .../emqx_bridge_impl_kafka_producer_SUITE.erl | 204 ++- mix.exs | 2 +- 18 files changed, 2214 insertions(+), 198 deletions(-) create mode 100644 changes/ee/feat-9564.en.md create mode 100644 changes/ee/feat-9564.zh.md create mode 100644 lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl create mode 100644 lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 37ec1266a..99a49f8fd 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -2,7 +2,7 @@ {application, emqx_bridge, [ {description, "EMQX bridges"}, {vsn, "0.1.13"}, - {registered, []}, + {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ kernel, diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 292369d36..5bc83dbd9 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -55,6 +55,7 @@ T == gcp_pubsub; T == influxdb_api_v1; T == influxdb_api_v2; + T == kafka_producer; T == redis_single; T == redis_sentinel; T == redis_cluster; @@ -137,12 +138,12 @@ load_hook(Bridges) -> maps:to_list(Bridges) ). -do_load_hook(Type, #{local_topic := _}) when ?EGRESS_DIR_BRIDGES(Type) -> +do_load_hook(Type, #{local_topic := LocalTopic}) when + ?EGRESS_DIR_BRIDGES(Type) andalso is_binary(LocalTopic) +-> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE); do_load_hook(mqtt, #{egress := #{local := #{topic := _}}}) -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE); -do_load_hook(kafka, #{producer := #{mqtt := #{topic := _}}}) -> - emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_BRIDGE); do_load_hook(_Type, _Conf) -> ok. @@ -223,6 +224,7 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnv) -> ]), ok = unload_hook(), ok = load_hook(NewConf), + ?tp(bridge_post_config_update_done, #{}), Result. list() -> @@ -406,9 +408,7 @@ get_matched_bridge_id(BType, Conf, Topic, BName, Acc) when ?EGRESS_DIR_BRIDGES(B do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) end; get_matched_bridge_id(mqtt, #{egress := #{local := #{topic := Filter}}}, Topic, BName, Acc) -> - do_get_matched_bridge_id(Topic, Filter, mqtt, BName, Acc); -get_matched_bridge_id(kafka, #{producer := #{mqtt := #{topic := Filter}}}, Topic, BName, Acc) -> - do_get_matched_bridge_id(Topic, Filter, kafka, BName, Acc). + do_get_matched_bridge_id(Topic, Filter, mqtt, BName, Acc). do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) -> case emqx_topic:match(Topic, Filter) of diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 53fc7df4c..f29115b0a 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -45,7 +45,12 @@ ]). %% bi-directional bridge with producer/consumer or ingress/egress configs --define(IS_BI_DIR_BRIDGE(TYPE), TYPE =:= <<"mqtt">>; TYPE =:= <<"kafka">>). +-define(IS_BI_DIR_BRIDGE(TYPE), + TYPE =:= <<"mqtt">> +). +-define(IS_INGRESS_BRIDGE(TYPE), + TYPE =:= <<"kafka_consumer">> orelse ?IS_BI_DIR_BRIDGE(TYPE) +). -if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt; @@ -297,12 +302,14 @@ parse_confs( max_retries => Retry } }; -parse_confs(Type, Name, Conf) when ?IS_BI_DIR_BRIDGE(Type) -> +parse_confs(Type, Name, Conf) when ?IS_INGRESS_BRIDGE(Type) -> %% For some drivers that can be used as data-sources, we need to provide a %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it %% receives a message from the external database. BId = bridge_id(Type, Name), Conf#{hookpoint => <<"$bridges/", BId/binary>>, bridge_name => Name}; +parse_confs(<<"kafka_producer">> = _Type, Name, Conf) -> + Conf#{bridge_name => Name}; parse_confs(_Type, _Name, Conf) -> Conf. diff --git a/apps/emqx_bridge/src/emqx_bridge_sup.erl b/apps/emqx_bridge/src/emqx_bridge_sup.erl index a5e72a8c6..46a87b74f 100644 --- a/apps/emqx_bridge/src/emqx_bridge_sup.erl +++ b/apps/emqx_bridge/src/emqx_bridge_sup.erl @@ -34,5 +34,3 @@ init([]) -> }, ChildSpecs = [], {ok, {SupFlags, ChildSpecs}}. - -%% internal functions diff --git a/changes/ee/feat-9564.en.md b/changes/ee/feat-9564.en.md new file mode 100644 index 000000000..860afbc63 --- /dev/null +++ b/changes/ee/feat-9564.en.md @@ -0,0 +1,3 @@ +Implemented Kafka Consumer bridge. +Now it's possible to consume messages from Kafka and publish them to +MQTT topics. diff --git a/changes/ee/feat-9564.zh.md b/changes/ee/feat-9564.zh.md new file mode 100644 index 000000000..8eb29998b --- /dev/null +++ b/changes/ee/feat-9564.zh.md @@ -0,0 +1,2 @@ +实现了Kafka消费者桥。 +现在可以从Kafka消费消息并将其发布到 MQTT主题。 diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index c41b95c3a..53bc5dddd 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -39,6 +39,16 @@ emqx_ee_bridge_kafka { zh: "桥接名字" } } + kafka_producer { + desc { + en: "Kafka Producer configuration." + zh: "Kafka Producer 配置。" + } + label { + en: "Kafka Producer" + zh: "Kafka Producer" + } + } producer_opts { desc { en: "Local MQTT data source and Kafka bridge configs." @@ -49,16 +59,6 @@ emqx_ee_bridge_kafka { zh: "MQTT 到 Kafka" } } - producer_mqtt_opts { - desc { - en: "MQTT data source. Optional when used as a rule-engine action." - zh: "需要桥接到 MQTT 源主题。" - } - label { - en: "MQTT Source Topic" - zh: "MQTT 源主题" - } - } mqtt_topic { desc { en: "MQTT topic or topic as data source (bridge input)." @@ -218,7 +218,7 @@ emqx_ee_bridge_kafka { } socket_nodelay { desc { - en: "When set to 'true', TCP buffer sent as soon as possible. " + en: "When set to 'true', TCP buffer is sent as soon as possible. " "Otherwise, the OS kernel may buffer small TCP packets for a while (40 ms by default)." zh: "设置‘true’让系统内核立即发送。否则当需要发送的内容很少时，可能会有一定延迟（默认 40 毫秒）。" } @@ -473,4 +473,128 @@ emqx_ee_bridge_kafka { zh: "GSSAPI/Kerberos" } } + + kafka_consumer { + desc { + en: "Kafka Consumer configuration." + zh: "Kafka Consumer的配置。" + } + label { + en: "Kafka Consumer" + zh: "Kafka Consumer" + } + } + consumer_opts { + desc { + en: "Local MQTT data sink and Kafka bridge configs." + zh: "本地MQTT数据汇和Kafka桥配置。" + } + label { + en: "MQTT to Kafka" + zh: "MQTT 到 Kafka" + } + } + consumer_kafka_opts { + desc { + en: "Kafka consumer configs." + zh: "Kafka消费者配置。" + } + label { + en: "Kafka Consumer" + zh: "卡夫卡消费者" + } + } + consumer_mqtt_opts { + desc { + en: "MQTT data sink." + zh: "MQTT数据汇。" + } + label { + en: "MQTT data sink." + zh: "MQTT数据汇。" + } + } + consumer_mqtt_topic { + desc { + en: "Local topic to which consumed Kafka messages should be published to." + zh: "消耗的Kafka消息应该被发布到的本地主题。" + } + label { + en: "MQTT Topic" + zh: "MQTT主题" + } + } + consumer_mqtt_qos { + desc { + en: "MQTT QoS used to publish messages consumed from Kafka." + zh: "MQTT QoS用于发布从Kafka消耗的消息。" + } + label { + en: "MQTT Topic QoS" + zh: "MQTT 主题服务质量" + } + } + consumer_mqtt_payload { + desc { + en: "The payload of the MQTT message to be published.\n" + "full_message will encode all data available as a JSON object," + "message_value will directly use the Kafka message value as the " + "MQTT message payload." + zh: "要发布的MQTT消息的有效载荷。" + "full_message将把所有可用数据编码为JSON对象，" + "message_value将直接使用Kafka消息值作为MQTT消息的有效载荷。" + } + label { + en: "MQTT Payload" + zh: "MQTT有效载荷" + } + } + consumer_kafka_topic { + desc { + en: "Kafka topic to consume from." + zh: "从Kafka主题消费。" + } + label { + en: "Kafka topic" + zh: "卡夫卡主题 " + } + } + consumer_max_batch_bytes { + desc { + en: "Maximum bytes to fetch in a batch of messages." + "NOTE: this value might be expanded to retry when " + "it is not enough to fetch even a single message, " + "then slowly shrink back to the given value." + zh: "在一批消息中要取的最大字节数。" + "注意：这个值可能会被扩大，" + "当它甚至不足以取到一条消息时，就会重试，" + "然后慢慢缩回到给定的值。" + } + label { + en: "Max Bytes" + zh: "最大字节数" + } + } + consumer_max_rejoin_attempts { + desc { + en: "Maximum number of times allowed for a member to re-join the group." + zh: "允许一个成员重新加入小组的最大次数。" + } + label { + en: "Max Rejoin Attempts" + zh: "最大的重新加入尝试" + } + } + consumer_offset_reset_policy { + desc { + en: "Defines how the consumers should reset the start offset when " + "a topic partition has and invalid or no initial offset." + zh: "定义当一个主题分区的初始偏移量无效或没有初始偏移量时，" + "消费者应如何重置开始偏移量。" + } + label { + en: "Offset Reset Policy" + zh: "偏移重置政策" + } + } } diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index fa6dd560e..be0cb5345 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -2,7 +2,7 @@ {deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.5"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} - , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}} + , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index ac181b251..6647ec212 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,7 +1,7 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, {vsn, "0.1.7"}, - {registered, []}, + {registered, [emqx_ee_bridge_kafka_consumer_sup]}, {applications, [ kernel, stdlib, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index b5c656291..2d7f5b5be 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -64,7 +64,8 @@ examples(Method) -> lists:foldl(Fun, #{}, schema_modules()). resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(kafka) -> emqx_bridge_impl_kafka; +resource_type(kafka_consumer) -> emqx_bridge_impl_kafka_consumer; +resource_type(kafka_producer) -> emqx_bridge_impl_kafka_producer; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; @@ -85,14 +86,6 @@ resource_type(dynamo) -> emqx_ee_connector_dynamo. fields(bridges) -> [ - {kafka, - mk( - hoconsc:map(name, ref(emqx_ee_bridge_kafka, "config")), - #{ - desc => <<"Kafka Bridge Config">>, - required => false - } - )}, {hstreamdb, mk( hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")), @@ -133,8 +126,8 @@ fields(bridges) -> required => false } )} - ] ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs() ++ - clickhouse_structs(). + ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ + pgsql_structs() ++ clickhouse_structs(). mongodb_structs() -> [ @@ -149,6 +142,16 @@ mongodb_structs() -> || Type <- [mongodb_rs, mongodb_sharded, mongodb_single] ]. +kafka_structs() -> + [ + {Type, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_kafka, Type)), + #{desc => <<"EMQX Enterprise Config">>, required => false} + )} + || Type <- [kafka_producer, kafka_consumer] + ]. + influxdb_structs() -> [ {Protocol, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index c345f6c74..865a5f64b 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -68,6 +68,10 @@ fields("put") -> fields("config"); fields("get") -> emqx_bridge_schema:status_fields() ++ fields("post"); +fields(kafka_producer) -> + fields("config") ++ fields(producer_opts); +fields(kafka_consumer) -> + fields("config") ++ fields(consumer_opts); fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, @@ -104,8 +108,6 @@ fields("config") -> mk(hoconsc:union([none, ref(auth_username_password), ref(auth_gssapi_kerberos)]), #{ default => none, desc => ?DESC("authentication") })}, - {producer, mk(hoconsc:union([none, ref(producer_opts)]), #{desc => ?DESC(producer_opts)})}, - %{consumer, mk(hoconsc:union([none, ref(consumer_opts)]), #{desc => ?DESC(consumer_opts)})}, {socket_opts, mk(ref(socket_opts), #{required => false, desc => ?DESC(socket_opts)})} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(auth_username_password) -> @@ -156,15 +158,16 @@ fields(socket_opts) -> ]; fields(producer_opts) -> [ - {mqtt, mk(ref(producer_mqtt_opts), #{desc => ?DESC(producer_mqtt_opts)})}, + %% Note: there's an implicit convention in `emqx_bridge' that, + %% for egress bridges with this config, the published messages + %% will be forwarded to such bridges. + {local_topic, mk(binary(), #{desc => ?DESC(mqtt_topic)})}, {kafka, mk(ref(producer_kafka_opts), #{ required => true, desc => ?DESC(producer_kafka_opts) })} ]; -fields(producer_mqtt_opts) -> - [{topic, mk(binary(), #{desc => ?DESC(mqtt_topic)})}]; fields(producer_kafka_opts) -> [ {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, @@ -241,24 +244,45 @@ fields(producer_buffer) -> default => false, desc => ?DESC(buffer_memory_overload_protection) })} + ]; +fields(consumer_opts) -> + [ + {kafka, + mk(ref(consumer_kafka_opts), #{required => true, desc => ?DESC(consumer_kafka_opts)})}, + {mqtt, mk(ref(consumer_mqtt_opts), #{required => true, desc => ?DESC(consumer_mqtt_opts)})} + ]; +fields(consumer_mqtt_opts) -> + [ + {topic, + mk(binary(), #{ + required => true, + desc => ?DESC(consumer_mqtt_topic) + })}, + {qos, mk(emqx_schema:qos(), #{default => 0, desc => ?DESC(consumer_mqtt_qos)})}, + {payload, + mk( + enum([full_message, message_value]), + #{default => full_message, desc => ?DESC(consumer_mqtt_payload)} + )} + ]; +fields(consumer_kafka_opts) -> + [ + {topic, mk(binary(), #{desc => ?DESC(consumer_kafka_topic)})}, + {max_batch_bytes, + mk(emqx_schema:bytesize(), #{ + default => "896KB", desc => ?DESC(consumer_max_batch_bytes) + })}, + {max_rejoin_attempts, + mk(non_neg_integer(), #{ + default => 5, desc => ?DESC(consumer_max_rejoin_attempts) + })}, + {offset_reset_policy, + mk( + enum([reset_to_latest, reset_to_earliest, reset_by_subscriber]), + #{default => reset_to_latest, desc => ?DESC(consumer_offset_reset_policy)} + )} ]. -% fields(consumer_opts) -> -% [ -% {kafka, mk(ref(consumer_kafka_opts), #{required => true, desc => ?DESC(consumer_kafka_opts)})}, -% {mqtt, mk(ref(consumer_mqtt_opts), #{required => true, desc => ?DESC(consumer_mqtt_opts)})} -% ]; -% fields(consumer_mqtt_opts) -> -% [ {topic, mk(string(), #{desc => ?DESC(consumer_mqtt_topic)})} -% ]; - -% fields(consumer_mqtt_opts) -> -% [ {topic, mk(string(), #{desc => ?DESC(consumer_mqtt_topic)})} -% ]; -% fields(consumer_kafka_opts) -> -% [ {topic, mk(string(), #{desc => ?DESC(consumer_kafka_topic)})} -% ]. - desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> @@ -272,11 +296,15 @@ struct_names() -> auth_gssapi_kerberos, auth_username_password, kafka_message, + kafka_producer, + kafka_consumer, producer_buffer, producer_kafka_opts, - producer_mqtt_opts, socket_opts, - producer_opts + producer_opts, + consumer_opts, + consumer_kafka_opts, + consumer_mqtt_opts ]. %% ------------------------------------------------------------------------------------------------- diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl index 49ca9fb86..c9dcce9a2 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl @@ -4,34 +4,38 @@ %% Kafka connection configuration -module(emqx_bridge_impl_kafka). --behaviour(emqx_resource). -%% callbacks of behaviour emqx_resource -export([ - callback_mode/0, - on_start/2, - on_stop/2, - on_query/3, - on_query_async/4, - on_get_status/2, - is_buffer_supported/0 + hosts/1, + make_client_id/2, + sasl/1 ]). -is_buffer_supported() -> true. +%% Parse comma separated host:port list into a [{Host,Port}] list +hosts(Hosts) when is_binary(Hosts) -> + hosts(binary_to_list(Hosts)); +hosts(Hosts) when is_list(Hosts) -> + kpro:parse_endpoints(Hosts). -callback_mode() -> async_if_possible. +%% Client ID is better to be unique to make it easier for Kafka side trouble shooting. +make_client_id(KafkaType0, BridgeName0) -> + KafkaType = to_bin(KafkaType0), + BridgeName = to_bin(BridgeName0), + iolist_to_binary([KafkaType, ":", BridgeName, ":", atom_to_list(node())]). -on_start(InstId, Config) -> - emqx_bridge_impl_kafka_producer:on_start(InstId, Config). +sasl(none) -> + undefined; +sasl(#{mechanism := Mechanism, username := Username, password := Password}) -> + {Mechanism, Username, emqx_secret:wrap(Password)}; +sasl(#{ + kerberos_principal := Principal, + kerberos_keytab_file := KeyTabFile +}) -> + {callback, brod_gssapi, {gssapi, KeyTabFile, Principal}}. -on_stop(InstId, State) -> - emqx_bridge_impl_kafka_producer:on_stop(InstId, State). - -on_query(InstId, Req, State) -> - emqx_bridge_impl_kafka_producer:on_query(InstId, Req, State). - -on_query_async(InstId, Req, ReplyFn, State) -> - emqx_bridge_impl_kafka_producer:on_query_async(InstId, Req, ReplyFn, State). - -on_get_status(InstId, State) -> - emqx_bridge_impl_kafka_producer:on_get_status(InstId, State). +to_bin(A) when is_atom(A) -> + atom_to_binary(A); +to_bin(L) when is_list(L) -> + list_to_binary(L); +to_bin(B) when is_binary(B) -> + B. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl new file mode 100644 index 000000000..f0480f2d6 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -0,0 +1,381 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_impl_kafka_consumer). + +-behaviour(emqx_resource). + +%% `emqx_resource' API +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_get_status/2 +]). + +%% `brod_group_consumer' API +-export([ + init/2, + handle_message/2 +]). + +-ifdef(TEST). +-export([consumer_group_id/1]). +-endif. + +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +%% needed for the #kafka_message record definition +-include_lib("brod/include/brod.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-type config() :: #{ + authentication := term(), + bootstrap_hosts := binary(), + bridge_name := atom(), + kafka := #{ + max_batch_bytes := emqx_schema:bytesize(), + max_rejoin_attempts := non_neg_integer(), + offset_reset_policy := offset_reset_policy(), + topic := binary() + }, + mqtt := #{ + topic := emqx_types:topic(), + qos := emqx_types:qos(), + payload := mqtt_payload() + }, + ssl := _, + any() => term() +}. +-type subscriber_id() :: emqx_ee_bridge_kafka_consumer_sup:child_id(). +-type state() :: #{ + kafka_topic := binary(), + subscriber_id := subscriber_id(), + kafka_client_id := brod:client_id() +}. +-type offset_reset_policy() :: reset_to_latest | reset_to_earliest | reset_by_subscriber. +-type mqtt_payload() :: full_message | message_value. +-type consumer_state() :: #{ + resource_id := resource_id(), + mqtt := #{ + payload := mqtt_payload(), + topic => emqx_types:topic(), + qos => emqx_types:qos() + }, + hookpoint := binary(), + kafka_topic := binary() +}. + +%%------------------------------------------------------------------------------------- +%% `emqx_resource' API +%%------------------------------------------------------------------------------------- + +callback_mode() -> + async_if_possible. + +%% there are no queries to be made to this bridge, so we say that +%% buffer is supported so we don't spawn unused resource buffer +%% workers. +is_buffer_supported() -> + true. + +-spec on_start(manager_id(), config()) -> {ok, state()}. +on_start(InstanceId, Config) -> + ensure_consumer_supervisor_started(), + #{ + authentication := Auth, + bootstrap_hosts := BootstrapHosts0, + bridge_name := BridgeName, + hookpoint := Hookpoint, + kafka := #{ + max_batch_bytes := MaxBatchBytes, + max_rejoin_attempts := MaxRejoinAttempts, + offset_reset_policy := OffsetResetPolicy, + topic := KafkaTopic + }, + mqtt := #{topic := MQTTTopic, qos := MQTTQoS, payload := MQTTPayload}, + ssl := SSL + } = Config, + BootstrapHosts = emqx_bridge_impl_kafka:hosts(BootstrapHosts0), + GroupConfig = [{max_rejoin_attempts, MaxRejoinAttempts}], + ConsumerConfig = [ + {max_bytes, MaxBatchBytes}, + {offset_reset_policy, OffsetResetPolicy} + ], + InitialState = #{ + resource_id => emqx_bridge_resource:resource_id(kafka_consumer, BridgeName), + mqtt => #{ + payload => MQTTPayload, + topic => MQTTTopic, + qos => MQTTQoS + }, + hookpoint => Hookpoint, + kafka_topic => KafkaTopic + }, + KafkaType = kafka_consumer, + %% Note: this is distinct per node. + ClientID0 = emqx_bridge_impl_kafka:make_client_id(KafkaType, BridgeName), + ClientID = binary_to_atom(ClientID0), + ClientOpts0 = + case Auth of + none -> []; + Auth -> [{sasl, emqx_bridge_impl_kafka:sasl(Auth)}] + end, + ClientOpts = add_ssl_opts(ClientOpts0, SSL), + case brod:start_client(BootstrapHosts, ClientID, ClientOpts) of + ok -> + ?tp( + kafka_consumer_client_started, + #{client_id => ClientID, instance_id => InstanceId} + ), + ?SLOG(info, #{ + msg => "kafka_consumer_client_started", + instance_id => InstanceId, + kafka_hosts => BootstrapHosts + }); + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_start_kafka_consumer_client", + instance_id => InstanceId, + kafka_hosts => BootstrapHosts, + reason => Reason + }), + throw(failed_to_start_kafka_client) + end, + %% note: the group id should be the same for all nodes in the + %% cluster, so that the load gets distributed between all + %% consumers and we don't repeat messages in the same cluster. + GroupID = consumer_group_id(BridgeName), + GroupSubscriberConfig = + #{ + client => ClientID, + group_id => GroupID, + topics => [KafkaTopic], + cb_module => ?MODULE, + init_data => InitialState, + message_type => message, + consumer_config => ConsumerConfig, + group_config => GroupConfig + }, + %% Below, we spawn a single `brod_group_consumer_v2' worker, with + %% no option for a pool of those. This is because that worker + %% spawns one worker for each assigned topic-partition + %% automatically, so we should not spawn duplicate workers. + SubscriberId = make_subscriber_id(BridgeName), + case emqx_ee_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of + {ok, _ConsumerPid} -> + ?tp( + kafka_consumer_subscriber_started, + #{instance_id => InstanceId, subscriber_id => SubscriberId} + ), + {ok, #{ + subscriber_id => SubscriberId, + kafka_client_id => ClientID, + kafka_topic => KafkaTopic + }}; + {error, Reason2} -> + ?SLOG(error, #{ + msg => "failed_to_start_kafka_consumer", + instance_id => InstanceId, + kafka_hosts => BootstrapHosts, + kafka_topic => KafkaTopic, + reason => Reason2 + }), + stop_client(ClientID), + throw(failed_to_start_kafka_consumer) + end. + +-spec on_stop(manager_id(), state()) -> ok. +on_stop(_InstanceID, State) -> + #{ + subscriber_id := SubscriberId, + kafka_client_id := ClientID + } = State, + stop_subscriber(SubscriberId), + stop_client(ClientID), + ok. + +-spec on_get_status(manager_id(), state()) -> connected | disconnected. +on_get_status(_InstanceID, State) -> + #{ + subscriber_id := SubscriberId, + kafka_client_id := ClientID, + kafka_topic := KafkaTopic + } = State, + case brod:get_partitions_count(ClientID, KafkaTopic) of + {ok, NPartitions} -> + do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions); + _ -> + disconnected + end. + +%%------------------------------------------------------------------------------------- +%% `brod_group_subscriber' API +%%------------------------------------------------------------------------------------- + +-spec init(_, consumer_state()) -> {ok, consumer_state()}. +init(_GroupData, State) -> + ?tp(kafka_consumer_subscriber_init, #{group_data => _GroupData, state => State}), + {ok, State}. + +-spec handle_message(#kafka_message{}, consumer_state()) -> {ok, commit, consumer_state()}. +handle_message(Message, State) -> + ?tp_span( + kafka_consumer_handle_message, + #{message => Message, state => State}, + begin + #{ + resource_id := ResourceId, + hookpoint := Hookpoint, + kafka_topic := KafkaTopic, + mqtt := #{ + topic := MQTTTopic, + payload := MQTTPayload, + qos := MQTTQoS + } + } = State, + FullMessage = #{ + offset => Message#kafka_message.offset, + key => Message#kafka_message.key, + value => Message#kafka_message.value, + ts => Message#kafka_message.ts, + ts_type => Message#kafka_message.ts_type, + headers => maps:from_list(Message#kafka_message.headers), + topic => KafkaTopic + }, + Payload = + case MQTTPayload of + full_message -> + FullMessage; + message_value -> + Message#kafka_message.value + end, + EncodedPayload = emqx_json:encode(Payload), + MQTTMessage = emqx_message:make(ResourceId, MQTTQoS, MQTTTopic, EncodedPayload), + _ = emqx:publish(MQTTMessage), + emqx:run_hook(Hookpoint, [FullMessage]), + emqx_resource_metrics:received_inc(ResourceId), + %% note: just `ack' does not commit the offset to the + %% kafka consumer group. + {ok, commit, State} + end + ). + +%%------------------------------------------------------------------------------------- +%% Helper fns +%%------------------------------------------------------------------------------------- + +add_ssl_opts(ClientOpts, #{enable := false}) -> + ClientOpts; +add_ssl_opts(ClientOpts, SSL) -> + [{ssl, emqx_tls_lib:to_client_opts(SSL)} | ClientOpts]. + +-spec make_subscriber_id(atom()) -> emqx_ee_bridge_kafka_consumer_sup:child_id(). +make_subscriber_id(BridgeName) -> + BridgeNameBin = atom_to_binary(BridgeName), + <<"kafka_subscriber:", BridgeNameBin/binary>>. + +ensure_consumer_supervisor_started() -> + Mod = emqx_ee_bridge_kafka_consumer_sup, + ChildSpec = + #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [Mod] + }, + case supervisor:start_child(emqx_bridge_sup, ChildSpec) of + {ok, _Pid} -> + ok; + {error, already_present} -> + ok; + {error, {already_started, _Pid}} -> + ok + end. + +-spec stop_subscriber(emqx_ee_bridge_kafka_consumer_sup:child_id()) -> ok. +stop_subscriber(SubscriberId) -> + _ = log_when_error( + fun() -> + emqx_ee_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId) + end, + #{ + msg => "failed_to_delete_kafka_subscriber", + subscriber_id => SubscriberId + } + ), + ok. + +-spec stop_client(brod:client_id()) -> ok. +stop_client(ClientID) -> + _ = log_when_error( + fun() -> + brod:stop_client(ClientID) + end, + #{ + msg => "failed_to_delete_kafka_consumer_client", + client_id => ClientID + } + ), + ok. + +-spec do_get_status(brod:client_id(), binary(), subscriber_id(), pos_integer()) -> + connected | disconnected. +do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions) -> + Results = + lists:map( + fun(N) -> + brod_client:get_leader_connection(ClientID, KafkaTopic, N) + end, + lists:seq(0, NPartitions - 1) + ), + AllLeadersOk = + length(Results) > 0 andalso + lists:all( + fun + ({ok, _}) -> + true; + (_) -> + false + end, + Results + ), + WorkersAlive = are_subscriber_workers_alive(SubscriberId), + case AllLeadersOk andalso WorkersAlive of + true -> + connected; + false -> + disconnected + end. + +are_subscriber_workers_alive(SubscriberId) -> + Children = supervisor:which_children(emqx_ee_bridge_kafka_consumer_sup), + case lists:keyfind(SubscriberId, 1, Children) of + false -> + false; + {_, Pid, _, _} -> + Workers = brod_group_subscriber_v2:get_workers(Pid), + %% we can't enforce the number of partitions on a single + %% node, as the group might be spread across an emqx + %% cluster. + lists:all(fun is_process_alive/1, maps:values(Workers)) + end. + +log_when_error(Fun, Log) -> + try + Fun() + catch + C:E -> + ?SLOG(error, Log#{ + exception => C, + reason => E + }) + end. + +-spec consumer_group_id(atom()) -> binary(). +consumer_group_id(BridgeName0) -> + BridgeName = atom_to_binary(BridgeName0), + <<"emqx-kafka-consumer:", BridgeName/binary>>. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index cff17b7de..785825a24 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -27,39 +27,31 @@ callback_mode() -> async_if_possible. %% @doc Config schema is defined in emqx_ee_bridge_kafka. on_start(InstId, Config) -> #{ - bridge_name := BridgeName, + authentication := Auth, bootstrap_hosts := Hosts0, + bridge_name := BridgeName, connect_timeout := ConnTimeout, + kafka := KafkaConfig = #{message := MessageTemplate, topic := KafkaTopic}, metadata_request_timeout := MetaReqTimeout, min_metadata_refresh_interval := MinMetaRefreshInterval, socket_opts := SocketOpts, - authentication := Auth, ssl := SSL } = Config, - %% TODO: change this to `kafka_producer` after refactoring for kafka_consumer - BridgeType = kafka, - ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), - _ = maybe_install_wolff_telemetry_handlers(ResourceID), - %% it's a bug if producer config is not found - %% the caller should not try to start a producer if - %% there is no producer config - ProducerConfigWrapper = get_required(producer, Config, no_kafka_producer_config), - ProducerConfig = get_required(kafka, ProducerConfigWrapper, no_kafka_producer_parameters), - MessageTemplate = get_required(message, ProducerConfig, no_kafka_message_template), - Hosts = hosts(Hosts0), - ClientId = make_client_id(BridgeName), + BridgeType = kafka_consumer, + ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + _ = maybe_install_wolff_telemetry_handlers(ResourceId), + Hosts = emqx_bridge_impl_kafka:hosts(Hosts0), + KafkaType = kafka_producer, + ClientId = emqx_bridge_impl_kafka:make_client_id(KafkaType, BridgeName), ClientConfig = #{ min_metadata_refresh_interval => MinMetaRefreshInterval, connect_timeout => ConnTimeout, client_id => ClientId, request_timeout => MetaReqTimeout, extra_sock_opts => socket_opts(SocketOpts), - sasl => sasl(Auth), + sasl => emqx_bridge_impl_kafka:sasl(Auth), ssl => ssl(SSL) }, - #{ - topic := KafkaTopic - } = ProducerConfig, case wolff:ensure_supervised_client(ClientId, Hosts, ClientConfig) of {ok, _} -> ?SLOG(info, #{ @@ -85,7 +77,7 @@ on_start(InstId, Config) -> _ -> string:equal(TestIdStart, InstId) end, - WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig, IsDryRun), + WolffProducerConfig = producers_config(BridgeName, ClientId, KafkaConfig, IsDryRun), case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of {ok, Producers} -> {ok, #{ @@ -93,7 +85,7 @@ on_start(InstId, Config) -> client_id => ClientId, kafka_topic => KafkaTopic, producers => Producers, - resource_id => ResourceID + resource_id => ResourceId }}; {error, Reason2} -> ?SLOG(error, #{ @@ -265,12 +257,6 @@ do_get_status(Client, KafkaTopic) -> disconnected end. -%% Parse comma separated host:port list into a [{Host,Port}] list -hosts(Hosts) when is_binary(Hosts) -> - hosts(binary_to_list(Hosts)); -hosts(Hosts) when is_list(Hosts) -> - kpro:parse_endpoints(Hosts). - %% Extra socket options, such as sndbuf size etc. socket_opts(Opts) when is_map(Opts) -> socket_opts(maps:to_list(Opts)); @@ -298,16 +284,6 @@ adjust_socket_buffer(Bytes, Opts) -> [{buffer, max(Bytes1, Bytes)} | Acc1] end. -sasl(none) -> - undefined; -sasl(#{mechanism := Mechanism, username := Username, password := Password}) -> - {Mechanism, Username, emqx_secret:wrap(Password)}; -sasl(#{ - kerberos_principal := Principal, - kerberos_keytab_file := KeyTabFile -}) -> - {callback, brod_gssapi, {gssapi, KeyTabFile, Principal}}. - ssl(#{enable := true} = SSL) -> emqx_tls_lib:to_client_opts(SSL); ssl(_) -> @@ -339,8 +315,7 @@ producers_config(BridgeName, ClientId, Input, IsDryRun) -> disk -> {false, replayq_dir(ClientId)}; hybrid -> {true, replayq_dir(ClientId)} end, - %% TODO: change this once we add kafka source - BridgeType = kafka, + BridgeType = kafka_producer, ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), #{ name => make_producer_name(BridgeName, IsDryRun), @@ -366,12 +341,6 @@ partitioner(key_dispatch) -> first_key_dispatch. replayq_dir(ClientId) -> filename:join([emqx:data_dir(), "kafka", ClientId]). -%% Client ID is better to be unique to make it easier for Kafka side trouble shooting. -make_client_id(BridgeName) when is_atom(BridgeName) -> - make_client_id(atom_to_list(BridgeName)); -make_client_id(BridgeName) -> - iolist_to_binary([BridgeName, ":", atom_to_list(node())]). - %% Producer name must be an atom which will be used as a ETS table name for %% partition worker lookup. make_producer_name(BridgeName, IsDryRun) when is_atom(BridgeName) -> @@ -400,11 +369,6 @@ with_log_at_error(Fun, Log) -> }) end. -get_required(Field, Config, Throw) -> - Value = maps:get(Field, Config, none), - Value =:= none andalso throw(Throw), - Value. - %% we *must* match the bridge id in the event metadata with that in %% the handler config; otherwise, multiple kafka producer bridges will %% install multiple handlers to the same wolff events, multiplying the diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl new file mode 100644 index 000000000..feec8c09b --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl @@ -0,0 +1,79 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_kafka_consumer_sup). + +-behaviour(supervisor). + +%% `supervisor' API +-export([init/1]). + +%% API +-export([ + start_link/0, + child_spec/2, + start_child/2, + ensure_child_deleted/1 +]). + +-type child_id() :: binary(). +-export_type([child_id/0]). + +%%-------------------------------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------------------------------- + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec child_spec(child_id(), map()) -> supervisor:child_spec(). +child_spec(Id, GroupSubscriberConfig) -> + Mod = brod_group_subscriber_v2, + #{ + id => Id, + start => {Mod, start_link, [GroupSubscriberConfig]}, + restart => permanent, + shutdown => 10_000, + type => worker, + modules => [Mod] + }. + +-spec start_child(child_id(), map()) -> {ok, pid()} | {error, term()}. +start_child(Id, GroupSubscriberConfig) -> + ChildSpec = child_spec(Id, GroupSubscriberConfig), + case supervisor:start_child(?MODULE, ChildSpec) of + {ok, Pid} -> + {ok, Pid}; + {ok, Pid, _Info} -> + {ok, Pid}; + {error, already_present} -> + supervisor:restart_child(?MODULE, Id); + {error, {already_started, Pid}} -> + {ok, Pid}; + {error, Error} -> + {error, Error} + end. + +-spec ensure_child_deleted(child_id()) -> ok. +ensure_child_deleted(Id) -> + case supervisor:terminate_child(?MODULE, Id) of + ok -> + ok = supervisor:delete_child(?MODULE, Id), + ok; + {error, not_found} -> + ok + end. + +%%-------------------------------------------------------------------------------------------- +%% `supervisor' API +%%-------------------------------------------------------------------------------------------- + +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 100, + period => 10 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl new file mode 100644 index 000000000..ce5003a8b --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -0,0 +1,1351 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_impl_kafka_consumer_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("brod/include/brod.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(BRIDGE_TYPE_BIN, <<"kafka_consumer">>). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, plain}, + {group, ssl}, + {group, sasl_plain}, + {group, sasl_ssl} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + SASLAuths = [ + sasl_auth_plain, + sasl_auth_scram256, + sasl_auth_scram512, + sasl_auth_kerberos + ], + SASLAuthGroups = [{group, Type} || Type <- SASLAuths], + OnlyOnceTCs = only_once_tests(), + MatrixTCs = AllTCs -- OnlyOnceTCs, + SASLTests = [{Group, MatrixTCs} || Group <- SASLAuths], + [ + {plain, MatrixTCs ++ OnlyOnceTCs}, + {ssl, MatrixTCs}, + {sasl_plain, SASLAuthGroups}, + {sasl_ssl, SASLAuthGroups} + ] ++ SASLTests. + +sasl_only_tests() -> + [t_failed_creation_then_fixed]. + +%% tests that do not need to be run on all groups +only_once_tests() -> + [ + t_bridge_rule_action_source, + t_cluster_group + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), + _ = application:stop(emqx_connector), + ok. + +init_per_group(plain = Type, Config) -> + KafkaHost = os:getenv("KAFKA_PLAIN_HOST", "toxiproxy.emqx.net"), + KafkaPort = list_to_integer(os:getenv("KAFKA_PLAIN_PORT", "9292")), + DirectKafkaHost = os:getenv("KAFKA_DIRECT_PLAIN_HOST", "kafka-1.emqx.net"), + DirectKafkaPort = list_to_integer(os:getenv("KAFKA_DIRECT_PLAIN_PORT", "9092")), + ProxyName = "kafka_plain", + case emqx_common_test_helpers:is_tcp_server_available(KafkaHost, KafkaPort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {kafka_host, KafkaHost}, + {kafka_port, KafkaPort}, + {direct_kafka_host, DirectKafkaHost}, + {direct_kafka_port, DirectKafkaPort}, + {kafka_type, Type}, + {use_sasl, false}, + {use_tls, false} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_kafka); + _ -> + {skip, no_kafka} + end + end; +init_per_group(sasl_plain = Type, Config) -> + KafkaHost = os:getenv("KAFKA_SASL_PLAIN_HOST", "toxiproxy.emqx.net"), + KafkaPort = list_to_integer(os:getenv("KAFKA_SASL_PLAIN_PORT", "9293")), + DirectKafkaHost = os:getenv("KAFKA_DIRECT_SASL_HOST", "kafka-1.emqx.net"), + DirectKafkaPort = list_to_integer(os:getenv("KAFKA_DIRECT_SASL_PORT", "9093")), + ProxyName = "kafka_sasl_plain", + case emqx_common_test_helpers:is_tcp_server_available(KafkaHost, KafkaPort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {kafka_host, KafkaHost}, + {kafka_port, KafkaPort}, + {direct_kafka_host, DirectKafkaHost}, + {direct_kafka_port, DirectKafkaPort}, + {kafka_type, Type}, + {use_sasl, true}, + {use_tls, false} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_kafka); + _ -> + {skip, no_kafka} + end + end; +init_per_group(ssl = Type, Config) -> + KafkaHost = os:getenv("KAFKA_SSL_HOST", "toxiproxy.emqx.net"), + KafkaPort = list_to_integer(os:getenv("KAFKA_SSL_PORT", "9294")), + DirectKafkaHost = os:getenv("KAFKA_DIRECT_SSL_HOST", "kafka-1.emqx.net"), + DirectKafkaPort = list_to_integer(os:getenv("KAFKA_DIRECT_SSL_PORT", "9094")), + ProxyName = "kafka_ssl", + case emqx_common_test_helpers:is_tcp_server_available(KafkaHost, KafkaPort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {kafka_host, KafkaHost}, + {kafka_port, KafkaPort}, + {direct_kafka_host, DirectKafkaHost}, + {direct_kafka_port, DirectKafkaPort}, + {kafka_type, Type}, + {use_sasl, false}, + {use_tls, true} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_kafka); + _ -> + {skip, no_kafka} + end + end; +init_per_group(sasl_ssl = Type, Config) -> + KafkaHost = os:getenv("KAFKA_SASL_SSL_HOST", "toxiproxy.emqx.net"), + KafkaPort = list_to_integer(os:getenv("KAFKA_SASL_SSL_PORT", "9295")), + DirectKafkaHost = os:getenv("KAFKA_DIRECT_SASL_SSL_HOST", "kafka-1.emqx.net"), + DirectKafkaPort = list_to_integer(os:getenv("KAFKA_DIRECT_SASL_SSL_PORT", "9095")), + ProxyName = "kafka_sasl_ssl", + case emqx_common_test_helpers:is_tcp_server_available(KafkaHost, KafkaPort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {kafka_host, KafkaHost}, + {kafka_port, KafkaPort}, + {direct_kafka_host, DirectKafkaHost}, + {direct_kafka_port, DirectKafkaPort}, + {kafka_type, Type}, + {use_sasl, true}, + {use_tls, true} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_kafka); + _ -> + {skip, no_kafka} + end + end; +init_per_group(sasl_auth_plain, Config) -> + [{sasl_auth_mechanism, plain} | Config]; +init_per_group(sasl_auth_scram256, Config) -> + [{sasl_auth_mechanism, scram_sha_256} | Config]; +init_per_group(sasl_auth_scram512, Config) -> + [{sasl_auth_mechanism, scram_sha_512} | Config]; +init_per_group(sasl_auth_kerberos, Config0) -> + %% currently it's tricky to setup kerberos + toxiproxy, probably + %% due to hostname issues... + UseTLS = ?config(use_tls, Config0), + {KafkaHost, KafkaPort} = + case UseTLS of + true -> + { + os:getenv("KAFKA_SASL_SSL_HOST", "kafka-1.emqx.net"), + list_to_integer(os:getenv("KAFKA_SASL_SSL_PORT", "9095")) + }; + false -> + { + os:getenv("KAFKA_SASL_PLAIN_HOST", "kafka-1.emqx.net"), + list_to_integer(os:getenv("KAFKA_SASL_PLAIN_PORT", "9093")) + } + end, + Config = + lists:map( + fun + ({kafka_host, _KafkaHost}) -> + {kafka_host, KafkaHost}; + ({kafka_port, _KafkaPort}) -> + {kafka_port, KafkaPort}; + (KV) -> + KV + end, + [{has_proxy, false}, {sasl_auth_mechanism, kerberos} | Config0] + ), + Config; +init_per_group(_Group, Config) -> + Config. + +common_init_per_group() -> + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + application:load(emqx_bridge), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), + {ok, _} = application:ensure_all_started(emqx_connector), + emqx_mgmt_api_test_util:init_suite(), + UniqueNum = integer_to_binary(erlang:unique_integer()), + MQTTTopic = <<"mqtt/topic/", UniqueNum/binary>>, + [ + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {mqtt_topic, MQTTTopic}, + {mqtt_qos, 0}, + {mqtt_payload, full_message}, + {num_partitions, 3} + ]. + +common_end_per_group(Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + ok. + +end_per_group(Group, Config) when + Group =:= plain; + Group =:= ssl; + Group =:= sasl_plain; + Group =:= sasl_ssl +-> + common_end_per_group(Config), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(TestCase, Config) when + TestCase =:= t_failed_creation_then_fixed +-> + KafkaType = ?config(kafka_type, Config), + AuthMechanism = ?config(sasl_auth_mechanism, Config), + IsSASL = lists:member(KafkaType, [sasl_plain, sasl_ssl]), + case {IsSASL, AuthMechanism} of + {true, kerberos} -> + [{skip_does_not_apply, true}]; + {true, _} -> + common_init_per_testcase(TestCase, Config); + {false, _} -> + [{skip_does_not_apply, true}] + end; +init_per_testcase(TestCase, Config) when + TestCase =:= t_failed_creation_then_fixed; + TestCase =:= t_on_get_status; + TestCase =:= t_receive_after_recovery +-> + HasProxy = proplists:get_value(has_proxy, Config, true), + case HasProxy of + false -> + [{skip_does_not_apply, true}]; + true -> + common_init_per_testcase(TestCase, Config) + end; +init_per_testcase(t_cluster_group = TestCase, Config0) -> + Config = emqx_misc:merge_opts(Config0, [{num_partitions, 6}]), + common_init_per_testcase(TestCase, Config); +init_per_testcase(TestCase, Config) -> + common_init_per_testcase(TestCase, Config). + +common_init_per_testcase(TestCase, Config0) -> + ct:timetrap(timer:seconds(60)), + delete_all_bridges(), + KafkaTopic = + << + (atom_to_binary(TestCase))/binary, + (integer_to_binary(erlang:unique_integer()))/binary + >>, + Config = [{kafka_topic, KafkaTopic} | Config0], + KafkaType = ?config(kafka_type, Config), + {Name, ConfigString, KafkaConfig} = kafka_config( + TestCase, KafkaType, Config + ), + ensure_topic(Config), + #{ + producers := Producers, + clientid := KafkaProducerClientId + } = start_producer(TestCase, Config), + ok = snabbkaffe:start_trace(), + [ + {kafka_name, Name}, + {kafka_config_string, ConfigString}, + {kafka_config, KafkaConfig}, + {kafka_producers, Producers}, + {kafka_producer_clientid, KafkaProducerClientId} + | Config + ]. + +end_per_testcase(_Testcase, Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok = snabbkaffe:stop(), + ok; + false -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + Producers = ?config(kafka_producers, Config), + KafkaProducerClientId = ?config(kafka_producer_clientid, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + ok = wolff:stop_and_delete_supervised_producers(Producers), + ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), + emqx_common_test_helpers:call_janitor(), + ok = snabbkaffe:stop(), + ok + end. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +start_producer(TestCase, Config) -> + KafkaTopic = ?config(kafka_topic, Config), + KafkaClientId = + <<"test-client-", (atom_to_binary(TestCase))/binary, + (integer_to_binary(erlang:unique_integer()))/binary>>, + DirectKafkaHost = ?config(direct_kafka_host, Config), + DirectKafkaPort = ?config(direct_kafka_port, Config), + UseTLS = ?config(use_tls, Config), + UseSASL = ?config(use_sasl, Config), + Hosts = emqx_bridge_impl_kafka:hosts( + DirectKafkaHost ++ ":" ++ integer_to_list(DirectKafkaPort) + ), + SSL = + case UseTLS of + true -> + %% hint: when running locally, need to + %% `chmod og+rw` those files to be readable. + emqx_tls_lib:to_client_opts( + #{ + keyfile => shared_secret(client_keyfile), + certfile => shared_secret(client_certfile), + cacertfile => shared_secret(client_cacertfile), + verify => verify_none, + enable => true + } + ); + false -> + [] + end, + SASL = + case UseSASL of + true -> {plain, <<"emqxuser">>, <<"password">>}; + false -> undefined + end, + ClientConfig = #{ + min_metadata_refresh_interval => 5_000, + connect_timeout => 5_000, + client_id => KafkaClientId, + request_timeout => 1_000, + sasl => SASL, + ssl => SSL + }, + {ok, Clients} = wolff:ensure_supervised_client(KafkaClientId, Hosts, ClientConfig), + ProducerConfig = + #{ + name => test_producer, + partitioner => roundrobin, + partition_count_refresh_interval_seconds => 1_000, + replayq_max_total_bytes => 10_000, + replayq_seg_bytes => 9_000, + drop_if_highmem => false, + required_acks => leader_only, + max_batch_bytes => 900_000, + max_send_ahead => 0, + compression => no_compression, + telemetry_meta_data => #{} + }, + {ok, Producers} = wolff:ensure_supervised_producers(KafkaClientId, KafkaTopic, ProducerConfig), + #{ + producers => Producers, + clients => Clients, + clientid => KafkaClientId + }. + +ensure_topic(Config) -> + KafkaTopic = ?config(kafka_topic, Config), + KafkaHost = ?config(kafka_host, Config), + KafkaPort = ?config(kafka_port, Config), + UseTLS = ?config(use_tls, Config), + UseSASL = ?config(use_sasl, Config), + NumPartitions = proplists:get_value(num_partitions, Config, 3), + Endpoints = [{KafkaHost, KafkaPort}], + TopicConfigs = [ + #{ + name => KafkaTopic, + num_partitions => NumPartitions, + replication_factor => 1, + assignments => [], + configs => [] + } + ], + RequestConfig = #{timeout => 5_000}, + ConnConfig0 = + case UseTLS of + true -> + %% hint: when running locally, need to + %% `chmod og+rw` those files to be readable. + #{ + ssl => emqx_tls_lib:to_client_opts( + #{ + keyfile => shared_secret(client_keyfile), + certfile => shared_secret(client_certfile), + cacertfile => shared_secret(client_cacertfile), + verify => verify_none, + enable => true + } + ) + }; + false -> + #{} + end, + ConnConfig = + case UseSASL of + true -> + ConnConfig0#{sasl => {plain, <<"emqxuser">>, <<"password">>}}; + false -> + ConnConfig0#{sasl => undefined} + end, + case brod:create_topics(Endpoints, TopicConfigs, RequestConfig, ConnConfig) of + ok -> ok; + {error, topic_already_exists} -> ok + end. + +shared_secret_path() -> + os:getenv("CI_SHARED_SECRET_PATH", "/var/lib/secret"). + +shared_secret(client_keyfile) -> + filename:join([shared_secret_path(), "client.key"]); +shared_secret(client_certfile) -> + filename:join([shared_secret_path(), "client.crt"]); +shared_secret(client_cacertfile) -> + filename:join([shared_secret_path(), "ca.crt"]); +shared_secret(rig_keytab) -> + filename:join([shared_secret_path(), "rig.keytab"]). + +publish(Config, Messages) -> + Producers = ?config(kafka_producers, Config), + ct:pal("publishing: ~p", [Messages]), + {_Partition, _OffsetReply} = wolff:send_sync(Producers, Messages, 10_000). + +kafka_config(TestCase, _KafkaType, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + KafkaHost = ?config(kafka_host, Config), + KafkaPort = ?config(kafka_port, Config), + KafkaTopic = ?config(kafka_topic, Config), + AuthType = proplists:get_value(sasl_auth_mechanism, Config, none), + UseTLS = proplists:get_value(use_tls, Config, false), + Name = << + (atom_to_binary(TestCase))/binary, UniqueNum/binary + >>, + MQTTTopic = proplists:get_value(mqtt_topic, Config, <<"mqtt/topic/", UniqueNum/binary>>), + MQTTQoS = proplists:get_value(mqtt_qos, Config, 0), + MQTTPayload = proplists:get_value(mqtt_payload, Config, full_message), + ConfigString = + io_lib:format( + "bridges.kafka_consumer.~s {\n" + " enable = true\n" + " bootstrap_hosts = \"~p:~b\"\n" + " connect_timeout = 5s\n" + " min_metadata_refresh_interval = 3s\n" + " metadata_request_timeout = 5s\n" + "~s" + " kafka {\n" + " topic = ~s\n" + " max_batch_bytes = 896KB\n" + " max_rejoin_attempts = 5\n" + %% todo: matrix this + " offset_reset_policy = reset_to_latest\n" + " }\n" + " mqtt {\n" + " topic = \"~s\"\n" + " qos = ~b\n" + " payload = ~p\n" + " }\n" + " ssl {\n" + " enable = ~p\n" + " verify = verify_none\n" + " server_name_indication = \"auto\"\n" + " }\n" + "}\n", + [ + Name, + KafkaHost, + KafkaPort, + authentication(AuthType), + KafkaTopic, + MQTTTopic, + MQTTQoS, + MQTTPayload, + UseTLS + ] + ), + {Name, ConfigString, parse_and_check(ConfigString, Name)}. + +authentication(Type) when + Type =:= scram_sha_256; + Type =:= scram_sha_512; + Type =:= plain +-> + io_lib:format( + " authentication = {\n" + " mechanism = ~p\n" + " username = emqxuser\n" + " password = password\n" + " }\n", + [Type] + ); +authentication(kerberos) -> + %% TODO: how to make this work locally outside docker??? + io_lib:format( + " authentication = {\n" + " kerberos_principal = rig@KDC.EMQX.NET\n" + " kerberos_keytab_file = \"~s\"\n" + " }\n", + [shared_secret(rig_keytab)] + ); +authentication(_) -> + " authentication = none\n". + +parse_and_check(ConfigString, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + TypeBin = ?BRIDGE_TYPE_BIN, + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{TypeBin := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(kafka_name, Config), + KafkaConfig0 = ?config(kafka_config, Config), + KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + emqx_bridge:create(Type, Name, KafkaConfig). + +delete_bridge(Config) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(kafka_name, Config), + emqx_bridge:remove(Type, Name). + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ). + +update_bridge_api(Config) -> + update_bridge_api(Config, _Overrides = #{}). + +update_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(kafka_name, Config), + KafkaConfig0 = ?config(kafka_config, Config), + KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), + Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("updating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of + {ok, Res0} -> {ok, emqx_json:decode(Res0, [return_maps])}; + Error -> Error + end, + ct:pal("bridge update result: ~p", [Res]), + Res. + +send_message(Config, Payload) -> + Name = ?config(kafka_name, Config), + Type = ?BRIDGE_TYPE_BIN, + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + emqx_bridge:send_message(BridgeId, Payload). + +resource_id(Config) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(kafka_name, Config), + emqx_bridge_resource:resource_id(Type, Name). + +instance_id(Config) -> + ResourceId = resource_id(Config), + [{_, InstanceId}] = ets:lookup(emqx_resource_manager, {owner, ResourceId}), + InstanceId. + +wait_for_expected_published_messages(Messages0, Timeout) -> + Messages = maps:from_list([{K, Msg} || Msg = #{key := K} <- Messages0]), + do_wait_for_expected_published_messages(Messages, [], Timeout). + +do_wait_for_expected_published_messages(Messages, Acc, _Timeout) when map_size(Messages) =:= 0 -> + lists:reverse(Acc); +do_wait_for_expected_published_messages(Messages0, Acc0, Timeout) -> + receive + {publish, Msg0 = #{payload := Payload}} -> + case emqx_json:safe_decode(Payload, [return_maps]) of + {error, _} -> + ct:pal("unexpected message: ~p; discarding", [Msg0]), + do_wait_for_expected_published_messages(Messages0, Acc0, Timeout); + {ok, Decoded = #{<<"key">> := K}} when is_map_key(K, Messages0) -> + Msg = Msg0#{payload := Decoded}, + ct:pal("received expected message: ~p", [Msg]), + Acc = [Msg | Acc0], + Messages = maps:remove(K, Messages0), + do_wait_for_expected_published_messages(Messages, Acc, Timeout); + {ok, Decoded} -> + ct:pal("unexpected message: ~p; discarding", [Msg0#{payload := Decoded}]), + do_wait_for_expected_published_messages(Messages0, Acc0, Timeout) + end + after Timeout -> + error( + {timed_out_waiting_for_published_messages, #{ + so_far => Acc0, + remaining => Messages0, + mailbox => process_info(self(), messages) + }} + ) + end. + +receive_published() -> + receive_published(#{}). + +receive_published(Opts0) -> + Default = #{n => 1, timeout => 10_000}, + Opts = maps:merge(Default, Opts0), + receive_published(Opts, []). + +receive_published(#{n := N, timeout := _Timeout}, Acc) when N =< 0 -> + lists:reverse(Acc); +receive_published(#{n := N, timeout := Timeout} = Opts, Acc) -> + receive + {publish, Msg} -> + receive_published(Opts#{n := N - 1}, [Msg | Acc]) + after Timeout -> + error( + {timeout, #{ + msgs_so_far => Acc, + mailbox => process_info(self(), messages), + expected_remaining => N + }} + ) + end. + +wait_until_subscribers_are_ready(N, Timeout) -> + {ok, _} = + snabbkaffe:block_until( + ?match_n_events(N, #{?snk_kind := kafka_consumer_subscriber_init}), + Timeout + ), + ok. + +%% kinda hacky, but for yet unknown reasons kafka/brod seem a bit +%% flaky about when they decide truly consuming the messages... +%% `Period' should be greater than the `sleep_timeout' of the consumer +%% (default 1 s). +ping_until_healthy(_Config, _Period, Timeout) when Timeout =< 0 -> + ct:fail("kafka subscriber did not stabilize!"); +ping_until_healthy(Config, Period, Timeout) -> + TimeA = erlang:monotonic_time(millisecond), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + publish(Config, [#{key => <<"probing">>, value => Payload}]), + Res = + ?block_until( + #{?snk_kind := kafka_consumer_handle_message, ?snk_span := {complete, _}}, + Period + ), + case Res of + timeout -> + TimeB = erlang:monotonic_time(millisecond), + ConsumedTime = TimeB - TimeA, + ping_until_healthy(Config, Period, Timeout - ConsumedTime); + {ok, _} -> + ResourceId = resource_id(Config), + emqx_resource_manager:reset_metrics(ResourceId), + ok + end. + +ensure_connected(Config) -> + ?retry( + _Interval = 500, + _NAttempts = 20, + {ok, _} = get_client_connection(Config) + ), + ok. + +consumer_clientid(Config) -> + KafkaName = ?config(kafka_name, Config), + binary_to_atom(emqx_bridge_impl_kafka:make_client_id(kafka_consumer, KafkaName)). + +get_client_connection(Config) -> + KafkaHost = ?config(kafka_host, Config), + KafkaPort = ?config(kafka_port, Config), + ClientID = consumer_clientid(Config), + brod_client:get_connection(ClientID, KafkaHost, KafkaPort). + +get_subscriber_workers() -> + [{_, SubscriberPid, _, _}] = supervisor:which_children(emqx_ee_bridge_kafka_consumer_sup), + brod_group_subscriber_v2:get_workers(SubscriberPid). + +wait_downs(Refs, _Timeout) when map_size(Refs) =:= 0 -> + ok; +wait_downs(Refs0, Timeout) -> + receive + {'DOWN', Ref, process, _Pid, _Reason} when is_map_key(Ref, Refs0) -> + Refs = maps:remove(Ref, Refs0), + wait_downs(Refs, Timeout) + after Timeout -> + ct:fail("processes didn't die; remaining: ~p", [map_size(Refs0)]) + end. + +create_rule_and_action_http(Config) -> + KafkaName = ?config(kafka_name, Config), + MQTTTopic = ?config(mqtt_topic, Config), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, KafkaName), + ActionFn = <<(atom_to_binary(?MODULE))/binary, ":action_response">>, + Params = #{ + enable => true, + sql => <<"SELECT * FROM \"$bridges/", BridgeId/binary, "\"">>, + actions => + [ + #{ + <<"function">> => <<"republish">>, + <<"args">> => + #{ + <<"topic">> => <<"republish/", MQTTTopic/binary>>, + <<"payload">> => <<>>, + <<"qos">> => 0, + <<"retain">> => false, + <<"user_properties">> => <<"${headers}">> + } + }, + #{<<"function">> => ActionFn} + ] + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + ct:pal("rule action params: ~p", [Params]), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +action_response(Selected, Envs, Args) -> + ?tp(action_response, #{ + selected => Selected, + envs => Envs, + args => Args + }), + ok. + +wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout) -> + do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, #{}). + +do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, Acc0) -> + case map_size(Acc0) =:= NPartitions of + true -> + {ok, Acc0}; + false -> + receive + {kafka_assignment, Node, {Pid, MemberId, GenerationId, TopicAssignments}} -> + Event = #{ + node => Node, + pid => Pid, + member_id => MemberId, + generation_id => GenerationId, + topic_assignments => TopicAssignments + }, + Acc = reconstruct_assignments_from_events(KafkaTopic, [Event], Acc0), + do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, Acc) + after Timeout -> + {timeout, Acc0} + end + end. + +reconstruct_assignments_from_events(KafkaTopic, Events) -> + reconstruct_assignments_from_events(KafkaTopic, Events, #{}). + +reconstruct_assignments_from_events(KafkaTopic, Events0, Acc0) -> + %% when running the test multiple times with the same kafka + %% cluster, kafka will send assignments from old test topics that + %% we must discard. + Assignments = [ + {MemberId, Node, P} + || #{ + node := Node, + member_id := MemberId, + topic_assignments := Assignments + } <- Events0, + #brod_received_assignment{topic = T, partition = P} <- Assignments, + T =:= KafkaTopic + ], + ct:pal("assignments for topic ~p:\n ~p", [KafkaTopic, Assignments]), + lists:foldl( + fun({MemberId, Node, Partition}, Acc) -> + Acc#{Partition => {Node, MemberId}} + end, + Acc0, + Assignments + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_start_and_consume_ok(Config) -> + MQTTTopic = ?config(mqtt_topic, Config), + MQTTQoS = ?config(mqtt_qos, Config), + KafkaTopic = ?config(kafka_topic, Config), + NPartitions = ?config(num_partitions, Config), + ResourceId = resource_id(Config), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + ?check_trace( + begin + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + wait_until_subscribers_are_ready(NPartitions, 40_000), + ping_until_healthy(Config, _Period = 1_500, _Timeout = 24_000), + {ok, C} = emqtt:start_link(), + on_exit(fun() -> emqtt:stop(C) end), + {ok, _} = emqtt:connect(C), + {ok, _, [0]} = emqtt:subscribe(C, MQTTTopic), + + {Res, {ok, _}} = + ?wait_async_action( + publish(Config, [ + #{ + key => <<"mykey">>, + value => Payload, + headers => [{<<"hkey">>, <<"hvalue">>}] + } + ]), + #{?snk_kind := kafka_consumer_handle_message, ?snk_span := {complete, _}}, + 20_000 + ), + Res + end, + fun({_Partition, OffsetReply}, Trace) -> + ?assertMatch([_, _ | _], ?of_kind(kafka_consumer_handle_message, Trace)), + Published = receive_published(), + ?assertMatch( + [ + #{ + qos := MQTTQoS, + topic := MQTTTopic, + payload := _ + } + ], + Published + ), + [#{payload := PayloadBin}] = Published, + ?assertMatch( + #{ + <<"value">> := Payload, + <<"key">> := <<"mykey">>, + <<"topic">> := KafkaTopic, + <<"offset">> := OffsetReply, + <<"headers">> := #{<<"hkey">> := <<"hvalue">>} + }, + emqx_json:decode(PayloadBin, [return_maps]), + #{ + offset_reply => OffsetReply, + kafka_topic => KafkaTopic, + payload => Payload + } + ), + ?assertEqual(1, emqx_resource_metrics:received_get(ResourceId)), + ok + end + ), + ok. + +t_on_get_status(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + do_t_on_get_status(Config) + end. + +do_t_on_get_status(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + KafkaName = ?config(kafka_name, Config), + ResourceId = emqx_bridge_resource:resource_id(kafka_consumer, KafkaName), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ct:sleep(1_200), + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ct:sleep(500), + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)) + end), + ok. + +%% ensure that we can create and use the bridge successfully after +%% creating it with bad config. +t_failed_creation_then_fixed(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + ?check_trace(do_t_failed_creation_then_fixed(Config), []) + end. + +do_t_failed_creation_then_fixed(Config) -> + ct:timetrap({seconds, 180}), + MQTTTopic = ?config(mqtt_topic, Config), + MQTTQoS = ?config(mqtt_qos, Config), + KafkaTopic = ?config(kafka_topic, Config), + NPartitions = ?config(num_partitions, Config), + {ok, _} = create_bridge(Config, #{ + <<"authentication">> => #{<<"password">> => <<"wrong password">>} + }), + ?retry( + _Interval0 = 200, + _Attempts0 = 10, + begin + ClientConn0 = get_client_connection(Config), + case ClientConn0 of + {error, client_down} -> + ok; + {error, {client_down, _Stacktrace}} -> + ok; + _ -> + error({client_should_be_down, ClientConn0}) + end + end + ), + %% now, update with the correct configuration + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + update_bridge_api(Config), + #{?snk_kind := kafka_consumer_subscriber_started}, + 60_000 + ) + ), + wait_until_subscribers_are_ready(NPartitions, 120_000), + ResourceId = resource_id(Config), + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), + ping_until_healthy(Config, _Period = 1_500, _Timeout = 24_000), + + {ok, C} = emqtt:start_link(), + on_exit(fun() -> emqtt:stop(C) end), + {ok, _} = emqtt:connect(C), + {ok, _, [0]} = emqtt:subscribe(C, MQTTTopic), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + + {_, {ok, _}} = + ?wait_async_action( + publish(Config, [ + #{ + key => <<"mykey">>, + value => Payload, + headers => [{<<"hkey">>, <<"hvalue">>}] + } + ]), + #{?snk_kind := kafka_consumer_handle_message, ?snk_span := {complete, _}}, + 20_000 + ), + Published = receive_published(), + ?assertMatch( + [ + #{ + qos := MQTTQoS, + topic := MQTTTopic, + payload := _ + } + ], + Published + ), + [#{payload := PayloadBin}] = Published, + ?assertMatch( + #{ + <<"value">> := Payload, + <<"key">> := <<"mykey">>, + <<"topic">> := KafkaTopic, + <<"offset">> := _, + <<"headers">> := #{<<"hkey">> := <<"hvalue">>} + }, + emqx_json:decode(PayloadBin, [return_maps]), + #{ + kafka_topic => KafkaTopic, + payload => Payload + } + ), + ok. + +%% check that we commit the offsets so that restarting an emqx node or +%% recovering from a network partition will make the subscribers +%% consume the messages produced during the down time. +t_receive_after_recovery(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + do_t_receive_after_recovery(Config) + end. + +do_t_receive_after_recovery(Config) -> + ct:timetrap(120_000), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + MQTTTopic = ?config(mqtt_topic, Config), + NPartitions = ?config(num_partitions, Config), + KafkaName = ?config(kafka_name, Config), + KafkaNameA = binary_to_atom(KafkaName), + KafkaClientId = consumer_clientid(Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + {ok, _} = create_bridge(Config), + ping_until_healthy(Config, _Period = 1_500, _Timeout0 = 24_000), + {ok, connected} = emqx_resource_manager:health_check(ResourceId), + %% 0) ensure each partition commits its offset so it can + %% recover later. + Messages0 = [ + #{ + key => <<"commit", (integer_to_binary(N))/binary>>, + value => <<"commit", (integer_to_binary(N))/binary>> + } + || N <- lists:seq(1, NPartitions) + ], + %% we do distinct passes over this producing part so that + %% wolff won't batch everything together. + lists:foreach( + fun(Msg) -> + {_, {ok, _}} = + ?wait_async_action( + publish(Config, [Msg]), + #{ + ?snk_kind := kafka_consumer_handle_message, + ?snk_span := {complete, {ok, commit, _}} + }, + _Timeout1 = 2_000 + ) + end, + Messages0 + ), + ?retry( + _Interval = 500, + _NAttempts = 20, + begin + GroupId = emqx_bridge_impl_kafka_consumer:consumer_group_id(KafkaNameA), + {ok, [#{partitions := Partitions}]} = brod:fetch_committed_offsets( + KafkaClientId, GroupId + ), + NPartitions = length(Partitions) + end + ), + %% we need some time to avoid flakiness due to the + %% subscription happening while the consumers are still + %% publishing messages... + ct:sleep(500), + + %% 1) cut the connection with kafka. + WorkerRefs = maps:from_list([ + {monitor(process, Pid), Pid} + || {_TopicPartition, Pid} <- + maps:to_list(get_subscriber_workers()) + ]), + NumMsgs = 50, + Messages1 = [ + begin + X = emqx_guid:to_hexstr(emqx_guid:gen()), + #{ + key => X, + value => X + } + end + || _ <- lists:seq(1, NumMsgs) + ], + {ok, C} = emqtt:start_link(), + on_exit(fun() -> emqtt:stop(C) end), + {ok, _} = emqtt:connect(C), + {ok, _, [0]} = emqtt:subscribe(C, MQTTTopic), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + wait_downs(WorkerRefs, _Timeout2 = 1_000), + %% 2) publish messages while the consumer is down. + %% we use `pmap' to avoid wolff sending the whole + %% batch to a single partition. + emqx_misc:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), + ok + end), + %% 3) restore and consume messages + {ok, SRef1} = snabbkaffe:subscribe( + ?match_event(#{ + ?snk_kind := kafka_consumer_handle_message, + ?snk_span := {complete, _} + }), + NumMsgs, + _Timeout3 = 60_000 + ), + {ok, _} = snabbkaffe:receive_events(SRef1), + #{num_msgs => NumMsgs, msgs => lists:sort(Messages1)} + end, + fun(#{num_msgs := NumMsgs, msgs := ExpectedMsgs}, Trace) -> + Received0 = wait_for_expected_published_messages(ExpectedMsgs, _Timeout4 = 2_000), + Received1 = + lists:map( + fun(#{payload := #{<<"key">> := K, <<"value">> := V}}) -> + #{key => K, value => V} + end, + Received0 + ), + Received = lists:sort(Received1), + ?assertEqual(ExpectedMsgs, Received), + ?assert(length(?of_kind(kafka_consumer_handle_message, Trace)) > NumMsgs * 2), + ok + end + ), + ok. + +t_bridge_rule_action_source(Config) -> + MQTTTopic = ?config(mqtt_topic, Config), + KafkaTopic = ?config(kafka_topic, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + {ok, _} = create_bridge(Config), + ping_until_healthy(Config, _Period = 1_500, _Timeout = 24_000), + + {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + + RepublishTopic = <<"republish/", MQTTTopic/binary>>, + {ok, C} = emqtt:start_link([{proto_ver, v5}]), + on_exit(fun() -> emqtt:stop(C) end), + {ok, _} = emqtt:connect(C), + {ok, _, [0]} = emqtt:subscribe(C, RepublishTopic), + + UniquePayload = emqx_guid:to_hexstr(emqx_guid:gen()), + {_, {ok, _}} = + ?wait_async_action( + publish(Config, [ + #{ + key => UniquePayload, + value => UniquePayload, + headers => [{<<"hkey">>, <<"hvalue">>}] + } + ]), + #{?snk_kind := action_response}, + 5_000 + ), + + #{republish_topic => RepublishTopic, unique_payload => UniquePayload} + end, + fun(Res, _Trace) -> + #{ + republish_topic := RepublishTopic, + unique_payload := UniquePayload + } = Res, + Published = receive_published(), + ?assertMatch( + [ + #{ + topic := RepublishTopic, + properties := #{'User-Property' := [{<<"hkey">>, <<"hvalue">>}]}, + payload := _Payload, + dup := false, + qos := 0, + retain := false + } + ], + Published + ), + [#{payload := RawPayload}] = Published, + ?assertMatch( + #{ + <<"key">> := UniquePayload, + <<"value">> := UniquePayload, + <<"headers">> := #{<<"hkey">> := <<"hvalue">>}, + <<"topic">> := KafkaTopic + }, + emqx_json:decode(RawPayload, [return_maps]) + ), + ?assertEqual(1, emqx_resource_metrics:received_get(ResourceId)), + ok + end + ), + ok. + +t_cluster_group(Config) -> + ct:timetrap({seconds, 180}), + TestPid = self(), + NPartitions = ?config(num_partitions, Config), + KafkaTopic = ?config(kafka_topic, Config), + KafkaName = ?config(kafka_name, Config), + ResourceId = resource_id(Config), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, KafkaName), + PrivDataDir = ?config(priv_dir, Config), + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, [emqx_conf, emqx_bridge, emqx_rule_engine]}, + {listener_ports, []}, + {peer_mod, slave}, + {priv_data_dir, PrivDataDir}, + {load_schema, true}, + {schema_mod, emqx_ee_conf_schema}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, []), + ok; + (emqx_conf) -> + ok; + (_) -> + ok + end} + ] + ), + ct:pal("cluster: ~p", [Cluster]), + ?check_trace( + begin + Nodes = + [_N1, N2 | _] = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + on_exit(fun() -> + lists:foreach( + fun(N) -> + ok = emqx_common_test_helpers:stop_slave(N) + end, + Nodes + ) + end), + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + ok = meck:new(brod_group_subscriber_v2, [ + passthrough, no_link, no_history, non_strict + ]), + ok = meck:expect( + brod_group_subscriber_v2, + assignments_received, + fun(Pid, MemberId, GenerationId, TopicAssignments) -> + TestPid ! + {kafka_assignment, node(), + {Pid, MemberId, GenerationId, TopicAssignments}}, + ?tp( + kafka_assignment, + #{ + node => node(), + pid => Pid, + member_id => MemberId, + generation_id => GenerationId, + topic_assignments => TopicAssignments + } + ), + meck:passthrough([Pid, MemberId, GenerationId, TopicAssignments]) + end + ), + ok + end) + end, + Nodes + ), + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := kafka_consumer_subscriber_started}), + length(Nodes), + 15_000 + ), + erpc:call(N2, fun() -> {ok, _} = create_bridge(Config) end), + {ok, _} = snabbkaffe:receive_events(SRef0), + lists:foreach( + fun(N) -> + ?assertMatch( + {ok, _}, + erpc:call(N, emqx_bridge, lookup, [BridgeId]), + #{node => N} + ) + end, + Nodes + ), + + %% give kafka some time to rebalance the group; we need to + %% sleep so that the two nodes have time to distribute the + %% subscribers, rather than just one node containing all + %% of them. + ct:sleep(10_000), + {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, 30_000), + lists:foreach( + fun(N) -> + ?assertEqual( + {ok, connected}, + erpc:call(N, emqx_resource_manager, health_check, [ResourceId]), + #{node => N} + ) + end, + Nodes + ), + + #{nodes => Nodes} + end, + fun(Res, Trace0) -> + #{nodes := Nodes} = Res, + Trace1 = ?of_kind(kafka_assignment, Trace0), + Assignments = reconstruct_assignments_from_events(KafkaTopic, Trace1), + ?assertEqual( + lists:usort(Nodes), + lists:usort([ + N + || {_Partition, {N, _MemberId}} <- + maps:to_list(Assignments) + ]) + ), + ?assertEqual(NPartitions, map_size(Assignments)), + ok + end + ), + ok. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 9b38e98d3..b1375e150 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -11,7 +11,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("brod/include/brod.hrl"). --define(PRODUCER, emqx_bridge_impl_kafka). +-define(PRODUCER, emqx_bridge_impl_kafka_producer). %%------------------------------------------------------------------------------ %% Things for REST API tests @@ -71,6 +71,10 @@ wait_until_kafka_is_up(Attempts) -> end. init_per_suite(Config) -> + %% ensure loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + application:load(emqx_bridge), ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), {ok, _} = application:ensure_all_started(emqx_connector), @@ -102,6 +106,13 @@ init_per_group(GroupName, Config) -> end_per_group(_, _) -> ok. +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + delete_all_bridges(), + ok. + set_special_configs(emqx_management) -> Listeners = #{http => #{port => 8081}}, Config = #{ @@ -222,7 +233,7 @@ kafka_bridge_rest_api_all_auth_methods(UseSSL) -> ok. kafka_bridge_rest_api_helper(Config) -> - BridgeType = "kafka", + BridgeType = "kafka_producer", BridgeName = "my_kafka_bridge", BridgeID = emqx_bridge_resource:bridge_id( erlang:list_to_binary(BridgeType), @@ -266,24 +277,18 @@ kafka_bridge_rest_api_helper(Config) -> %% Create new Kafka bridge KafkaTopic = "test-topic-one-partition", CreateBodyTmp = #{ - <<"type">> => <<"kafka">>, + <<"type">> => <<"kafka_producer">>, <<"name">> => <<"my_kafka_bridge">>, <<"bootstrap_hosts">> => iolist_to_binary(maps:get(<<"bootstrap_hosts">>, Config)), <<"enable">> => true, <<"authentication">> => maps:get(<<"authentication">>, Config), - <<"producer">> => #{ - <<"mqtt">> => #{ - topic => <<"t/#">> - }, - <<"kafka">> => #{ - <<"topic">> => iolist_to_binary(KafkaTopic), - <<"buffer">> => #{ - <<"memory_overload_protection">> => <<"false">> - }, - <<"message">> => #{ - <<"key">> => <<"${clientid}">>, - <<"value">> => <<"${.payload}">> - } + <<"local_topic">> => <<"t/#">>, + <<"kafka">> => #{ + <<"topic">> => iolist_to_binary(KafkaTopic), + <<"buffer">> => #{<<"memory_overload_protection">> => <<"false">>}, + <<"message">> => #{ + <<"key">> => <<"${clientid}">>, + <<"value">> => <<"${.payload}">> } } }, @@ -355,6 +360,7 @@ kafka_bridge_rest_api_helper(Config) -> %% Cleanup {ok, 204, _} = show(http_delete(BridgesPartsIdDeleteAlsoActions)), false = MyKafkaBridgeExists(), + delete_all_bridges(), ok. %%------------------------------------------------------------------------------ @@ -371,9 +377,10 @@ t_failed_creation_then_fix(Config) -> ValidAuthSettings = valid_sasl_plain_settings(), WrongAuthSettings = ValidAuthSettings#{"password" := "wrong"}, Hash = erlang:phash2([HostsString, ?FUNCTION_NAME]), + Type = kafka_producer, Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash), - ResourceId = emqx_bridge_resource:resource_id("kafka", Name), - BridgeId = emqx_bridge_resource:bridge_id("kafka", Name), + ResourceId = emqx_bridge_resource:resource_id("kafka_producer", Name), + BridgeId = emqx_bridge_resource:bridge_id("kafka_producer", Name), KafkaTopic = "test-topic-one-partition", WrongConf = config(#{ "authentication" => WrongAuthSettings, @@ -397,15 +404,19 @@ t_failed_creation_then_fix(Config) -> "ssl" => #{} }), %% creates, but fails to start producers - %% FIXME: change to kafka_producer after config refactoring - ?assertMatch(ok, emqx_bridge_resource:create(kafka, erlang:list_to_atom(Name), WrongConf, #{})), - ?assertThrow(failed_to_start_kafka_producer, ?PRODUCER:on_start(ResourceId, WrongConf)), + {ok, #{config := WrongConfigAtom1}} = emqx_bridge:create( + Type, erlang:list_to_atom(Name), WrongConf + ), + WrongConfigAtom = WrongConfigAtom1#{bridge_name => Name}, + ?assertThrow(failed_to_start_kafka_producer, ?PRODUCER:on_start(ResourceId, WrongConfigAtom)), %% before throwing, it should cleanup the client process. ?assertEqual([], supervisor:which_children(wolff_client_sup)), - %% FIXME: change to kafka_producer after config refactoring %% must succeed with correct config - ?assertMatch(ok, emqx_bridge_resource:create(kafka, erlang:list_to_atom(Name), ValidConf, #{})), - {ok, State} = ?PRODUCER:on_start(ResourceId, ValidConf), + {ok, #{config := ValidConfigAtom1}} = emqx_bridge:create( + Type, erlang:list_to_atom(Name), ValidConf + ), + ValidConfigAtom = ValidConfigAtom1#{bridge_name => Name}, + {ok, State} = ?PRODUCER:on_start(ResourceId, ValidConfigAtom), %% To make sure we get unique value timer:sleep(1), Time = erlang:monotonic_time(), @@ -423,6 +434,7 @@ t_failed_creation_then_fix(Config) -> %% TODO: refactor those into init/end per testcase ok = ?PRODUCER:on_stop(ResourceId, State), ok = emqx_bridge_resource:remove(BridgeId), + delete_all_bridges(), ok. %%------------------------------------------------------------------------------ @@ -487,6 +499,7 @@ publish_helper( }, Conf0 ) -> + delete_all_bridges(), HostsString = case {AuthSettings, SSLSettings} of {"none", Map} when map_size(Map) =:= 0 -> @@ -500,8 +513,8 @@ publish_helper( end, Hash = erlang:phash2([HostsString, AuthSettings, SSLSettings]), Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash), - InstId = emqx_bridge_resource:resource_id("kafka", Name), - BridgeId = emqx_bridge_resource:bridge_id("kafka", Name), + Type = "kafka_producer", + InstId = emqx_bridge_resource:resource_id(Type, Name), KafkaTopic = "test-topic-one-partition", Conf = config( #{ @@ -509,30 +522,40 @@ publish_helper( "kafka_hosts_string" => HostsString, "kafka_topic" => KafkaTopic, "instance_id" => InstId, + "local_topic" => <<"mqtt/local">>, "ssl" => SSLSettings }, Conf0 ), - - emqx_bridge_resource:create(kafka, erlang:list_to_atom(Name), Conf, #{}), + {ok, _} = emqx_bridge:create( + <<"kafka_producer">>, list_to_binary(Name), Conf + ), %% To make sure we get unique value timer:sleep(1), Time = erlang:monotonic_time(), BinTime = integer_to_binary(Time), + Partition = 0, Msg = #{ clientid => BinTime, payload => <<"payload">>, timestamp => Time }, - {ok, Offset} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, 0), - ct:pal("base offset before testing ~p", [Offset]), - StartRes = ?PRODUCER:on_start(InstId, Conf), - {ok, State} = StartRes, + {ok, Offset0} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, Partition), + ct:pal("base offset before testing ~p", [Offset0]), + {ok, _Group, #{state := State}} = emqx_resource:get_instance(InstId), ok = send(CtConfig, InstId, Msg, State), - {ok, {_, [KafkaMsg]}} = brod:fetch(kafka_hosts(), KafkaTopic, 0, Offset), - ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg), - ok = ?PRODUCER:on_stop(InstId, State), - ok = emqx_bridge_resource:remove(BridgeId), + {ok, {_, [KafkaMsg0]}} = brod:fetch(kafka_hosts(), KafkaTopic, Partition, Offset0), + ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg0), + + %% test that it forwards from local mqtt topic as well + {ok, Offset1} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, Partition), + ct:pal("base offset before testing (2) ~p", [Offset1]), + emqx:publish(emqx_message:make(<<"mqtt/local">>, <<"payload">>)), + ct:sleep(2_000), + {ok, {_, [KafkaMsg1]}} = brod:fetch(kafka_hosts(), KafkaTopic, Partition, Offset1), + ?assertMatch(#kafka_message{value = <<"payload">>}, KafkaMsg1), + + delete_all_bridges(), ok. default_config() -> @@ -545,18 +568,24 @@ config(Args0, More) -> Args1 = maps:merge(default_config(), Args0), Args = maps:merge(Args1, More), ConfText = hocon_config(Args), - ct:pal("Running tests with conf:\n~s", [ConfText]), - {ok, Conf} = hocon:binary(ConfText), - #{config := Parsed} = hocon_tconf:check_plain( - emqx_ee_bridge_kafka, - #{<<"config">> => Conf}, - #{atom_key => true} - ), + {ok, Conf} = hocon:binary(ConfText, #{format => map}), + ct:pal("Running tests with conf:\n~p", [Conf]), InstId = maps:get("instance_id", Args), <<"bridge:", BridgeId/binary>> = InstId, - Parsed#{bridge_name => erlang:element(2, emqx_bridge_resource:parse_bridge_id(BridgeId))}. + {Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeId), + TypeBin = atom_to_binary(Type), + hocon_tconf:check_plain( + emqx_bridge_schema, + Conf, + #{atom_key => false, required => false} + ), + #{<<"bridges">> := #{TypeBin := #{Name := Parsed}}} = Conf, + Parsed. hocon_config(Args) -> + InstId = maps:get("instance_id", Args), + <<"bridge:", BridgeId/binary>> = InstId, + {_Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeId), AuthConf = maps:get("authentication", Args), AuthTemplate = iolist_to_binary(hocon_config_template_authentication(AuthConf)), AuthConfRendered = bbmustache:render(AuthTemplate, AuthConf), @@ -567,6 +596,7 @@ hocon_config(Args) -> iolist_to_binary(hocon_config_template()), Args#{ "authentication" => AuthConfRendered, + "bridge_name" => Name, "ssl" => SSLConfRendered } ), @@ -575,22 +605,30 @@ hocon_config(Args) -> %% erlfmt-ignore hocon_config_template() -> """ -bootstrap_hosts = \"{{ kafka_hosts_string }}\" -enable = true -authentication = {{{ authentication }}} -ssl = {{{ ssl }}} -producer = { - mqtt { - topic = \"t/#\" +bridges.kafka_producer.{{ bridge_name }} { + bootstrap_hosts = \"{{ kafka_hosts_string }}\" + enable = true + authentication = {{{ authentication }}} + ssl = {{{ ssl }}} + local_topic = \"{{ local_topic }}\" + kafka = { + message = { + key = \"${clientid}\" + value = \"${.payload}\" + timestamp = \"${timestamp}\" } - kafka = { - topic = \"{{ kafka_topic }}\" - message = {key = \"${clientid}\", value = \"${.payload}\"} - partition_strategy = {{ partition_strategy }} - buffer = { - memory_overload_protection = false - } + buffer = { + memory_overload_protection = false } + partition_strategy = {{ partition_strategy }} + topic = \"{{ kafka_topic }}\" + } + metadata_request_timeout = 5s + min_metadata_refresh_interval = 3s + socket_opts { + nodelay = true + } + connect_timeout = 5s } """. @@ -631,22 +669,42 @@ hocon_config_template_ssl(_) -> """. kafka_hosts_string() -> - "kafka-1.emqx.net:9092,". + KafkaHost = os:getenv("KAFKA_PLAIN_HOST", "kafka-1.emqx.net"), + KafkaPort = os:getenv("KAFKA_PLAIN_PORT", "9092"), + KafkaHost ++ ":" ++ KafkaPort ++ ",". kafka_hosts_string_sasl() -> - "kafka-1.emqx.net:9093,". + KafkaHost = os:getenv("KAFKA_SASL_PLAIN_HOST", "kafka-1.emqx.net"), + KafkaPort = os:getenv("KAFKA_SASL_PLAIN_PORT", "9093"), + KafkaHost ++ ":" ++ KafkaPort ++ ",". kafka_hosts_string_ssl() -> - "kafka-1.emqx.net:9094,". + KafkaHost = os:getenv("KAFKA_SSL_HOST", "kafka-1.emqx.net"), + KafkaPort = os:getenv("KAFKA_SSL_PORT", "9094"), + KafkaHost ++ ":" ++ KafkaPort ++ ",". kafka_hosts_string_ssl_sasl() -> - "kafka-1.emqx.net:9095,". + KafkaHost = os:getenv("KAFKA_SASL_SSL_HOST", "kafka-1.emqx.net"), + KafkaPort = os:getenv("KAFKA_SASL_SSL_PORT", "9095"), + KafkaHost ++ ":" ++ KafkaPort ++ ",". + +shared_secret_path() -> + os:getenv("CI_SHARED_SECRET_PATH", "/var/lib/secret"). + +shared_secret(client_keyfile) -> + filename:join([shared_secret_path(), "client.key"]); +shared_secret(client_certfile) -> + filename:join([shared_secret_path(), "client.crt"]); +shared_secret(client_cacertfile) -> + filename:join([shared_secret_path(), "ca.crt"]); +shared_secret(rig_keytab) -> + filename:join([shared_secret_path(), "rig.keytab"]). valid_ssl_settings() -> #{ - "cacertfile" => <<"/var/lib/secret/ca.crt">>, - "certfile" => <<"/var/lib/secret/client.crt">>, - "keyfile" => <<"/var/lib/secret/client.key">>, + "cacertfile" => shared_secret(client_cacertfile), + "certfile" => shared_secret(client_certfile), + "keyfile" => shared_secret(client_keyfile), "enable" => <<"true">> }. @@ -670,7 +728,7 @@ valid_sasl_scram512_settings() -> valid_sasl_kerberos_settings() -> #{ "kerberos_principal" => "rig@KDC.EMQX.NET", - "kerberos_keytab_file" => "/var/lib/secret/rig.keytab" + "kerberos_keytab_file" => shared_secret(rig_keytab) }. kafka_hosts() -> @@ -732,3 +790,17 @@ api_path(Parts) -> json(Data) -> {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), Jsx. + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ), + %% at some point during the tests, sometimes `emqx_bridge:list()' + %% returns an empty list, but `emqx:get_config([bridges])' returns + %% a bunch of orphan test bridges... + lists:foreach(fun emqx_resource:remove/1, emqx_resource:list_instances()), + emqx_config:put([bridges], #{}), + ok. diff --git a/mix.exs b/mix.exs index 4f4174ee2..42354f8dc 100644 --- a/mix.exs +++ b/mix.exs @@ -135,7 +135,7 @@ defmodule EMQXUmbrella.MixProject do {:wolff, github: "kafka4beam/wolff", tag: "1.7.5"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"}, - {:brod, github: "kafka4beam/brod", tag: "3.16.7"}, + {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.8", override: true}, {:supervisor3, "1.1.11", override: true} ] From 65c15b3faeb56268482c81e057490ef547f96357 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 1 Mar 2023 16:12:24 -0300 Subject: [PATCH 56/88] refactor(kafka_consumer): move subscriber startup logic to separate fn --- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 144 ++++++++++-------- 1 file changed, 80 insertions(+), 64 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index f0480f2d6..076d7fd97 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -82,37 +82,21 @@ is_buffer_supported() -> -spec on_start(manager_id(), config()) -> {ok, state()}. on_start(InstanceId, Config) -> - ensure_consumer_supervisor_started(), #{ authentication := Auth, bootstrap_hosts := BootstrapHosts0, bridge_name := BridgeName, - hookpoint := Hookpoint, + hookpoint := _, kafka := #{ - max_batch_bytes := MaxBatchBytes, - max_rejoin_attempts := MaxRejoinAttempts, - offset_reset_policy := OffsetResetPolicy, - topic := KafkaTopic + max_batch_bytes := _, + max_rejoin_attempts := _, + offset_reset_policy := _, + topic := _ }, - mqtt := #{topic := MQTTTopic, qos := MQTTQoS, payload := MQTTPayload}, + mqtt := #{topic := _, qos := _, payload := _}, ssl := SSL } = Config, BootstrapHosts = emqx_bridge_impl_kafka:hosts(BootstrapHosts0), - GroupConfig = [{max_rejoin_attempts, MaxRejoinAttempts}], - ConsumerConfig = [ - {max_bytes, MaxBatchBytes}, - {offset_reset_policy, OffsetResetPolicy} - ], - InitialState = #{ - resource_id => emqx_bridge_resource:resource_id(kafka_consumer, BridgeName), - mqtt => #{ - payload => MQTTPayload, - topic => MQTTTopic, - qos => MQTTQoS - }, - hookpoint => Hookpoint, - kafka_topic => KafkaTopic - }, KafkaType = kafka_consumer, %% Note: this is distinct per node. ClientID0 = emqx_bridge_impl_kafka:make_client_id(KafkaType, BridgeName), @@ -143,48 +127,7 @@ on_start(InstanceId, Config) -> }), throw(failed_to_start_kafka_client) end, - %% note: the group id should be the same for all nodes in the - %% cluster, so that the load gets distributed between all - %% consumers and we don't repeat messages in the same cluster. - GroupID = consumer_group_id(BridgeName), - GroupSubscriberConfig = - #{ - client => ClientID, - group_id => GroupID, - topics => [KafkaTopic], - cb_module => ?MODULE, - init_data => InitialState, - message_type => message, - consumer_config => ConsumerConfig, - group_config => GroupConfig - }, - %% Below, we spawn a single `brod_group_consumer_v2' worker, with - %% no option for a pool of those. This is because that worker - %% spawns one worker for each assigned topic-partition - %% automatically, so we should not spawn duplicate workers. - SubscriberId = make_subscriber_id(BridgeName), - case emqx_ee_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of - {ok, _ConsumerPid} -> - ?tp( - kafka_consumer_subscriber_started, - #{instance_id => InstanceId, subscriber_id => SubscriberId} - ), - {ok, #{ - subscriber_id => SubscriberId, - kafka_client_id => ClientID, - kafka_topic => KafkaTopic - }}; - {error, Reason2} -> - ?SLOG(error, #{ - msg => "failed_to_start_kafka_consumer", - instance_id => InstanceId, - kafka_hosts => BootstrapHosts, - kafka_topic => KafkaTopic, - reason => Reason2 - }), - stop_client(ClientID), - throw(failed_to_start_kafka_consumer) - end. + start_subscriber(Config, InstanceId, ClientID). -spec on_stop(manager_id(), state()) -> ok. on_stop(_InstanceID, State) -> @@ -296,6 +239,79 @@ ensure_consumer_supervisor_started() -> ok end. +-spec start_subscriber(config(), manager_id(), brod:client_id()) -> {ok, state()}. +start_subscriber(Config, InstanceId, ClientID) -> + #{ + bootstrap_hosts := BootstrapHosts0, + bridge_name := BridgeName, + hookpoint := Hookpoint, + kafka := #{ + max_batch_bytes := MaxBatchBytes, + max_rejoin_attempts := MaxRejoinAttempts, + offset_reset_policy := OffsetResetPolicy, + topic := KafkaTopic + }, + mqtt := #{topic := MQTTTopic, qos := MQTTQoS, payload := MQTTPayload} + } = Config, + ensure_consumer_supervisor_started(), + InitialState = #{ + resource_id => emqx_bridge_resource:resource_id(kafka_consumer, BridgeName), + mqtt => #{ + payload => MQTTPayload, + topic => MQTTTopic, + qos => MQTTQoS + }, + hookpoint => Hookpoint, + kafka_topic => KafkaTopic + }, + %% note: the group id should be the same for all nodes in the + %% cluster, so that the load gets distributed between all + %% consumers and we don't repeat messages in the same cluster. + GroupID = consumer_group_id(BridgeName), + ConsumerConfig = [ + {max_bytes, MaxBatchBytes}, + {offset_reset_policy, OffsetResetPolicy} + ], + GroupConfig = [{max_rejoin_attempts, MaxRejoinAttempts}], + GroupSubscriberConfig = + #{ + client => ClientID, + group_id => GroupID, + topics => [KafkaTopic], + cb_module => ?MODULE, + init_data => InitialState, + message_type => message, + consumer_config => ConsumerConfig, + group_config => GroupConfig + }, + %% Below, we spawn a single `brod_group_consumer_v2' worker, with + %% no option for a pool of those. This is because that worker + %% spawns one worker for each assigned topic-partition + %% automatically, so we should not spawn duplicate workers. + SubscriberId = make_subscriber_id(BridgeName), + case emqx_ee_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of + {ok, _ConsumerPid} -> + ?tp( + kafka_consumer_subscriber_started, + #{instance_id => InstanceId, subscriber_id => SubscriberId} + ), + {ok, #{ + subscriber_id => SubscriberId, + kafka_client_id => ClientID, + kafka_topic => KafkaTopic + }}; + {error, Reason2} -> + ?SLOG(error, #{ + msg => "failed_to_start_kafka_consumer", + instance_id => InstanceId, + kafka_hosts => emqx_bridge_impl_kafka:hosts(BootstrapHosts0), + kafka_topic => KafkaTopic, + reason => Reason2 + }), + stop_client(ClientID), + throw(failed_to_start_kafka_consumer) + end. + -spec stop_subscriber(emqx_ee_bridge_kafka_consumer_sup:child_id()) -> ok. stop_subscriber(SubscriberId) -> _ = log_when_error( From 1d5fe14a307d3c5af5ca35020f882036cbbe843b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 1 Mar 2023 16:16:49 -0300 Subject: [PATCH 57/88] test: remove sleeps --- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index b1375e150..7a214de08 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -417,9 +417,7 @@ t_failed_creation_then_fix(Config) -> ), ValidConfigAtom = ValidConfigAtom1#{bridge_name => Name}, {ok, State} = ?PRODUCER:on_start(ResourceId, ValidConfigAtom), - %% To make sure we get unique value - timer:sleep(1), - Time = erlang:monotonic_time(), + Time = erlang:unique_integer(), BinTime = integer_to_binary(Time), Msg = #{ clientid => BinTime, @@ -530,9 +528,7 @@ publish_helper( {ok, _} = emqx_bridge:create( <<"kafka_producer">>, list_to_binary(Name), Conf ), - %% To make sure we get unique value - timer:sleep(1), - Time = erlang:monotonic_time(), + Time = erlang:unique_integer(), BinTime = integer_to_binary(Time), Partition = 0, Msg = #{ From e1fdd041b3b8279a4425868f8319f7065c339c14 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 1 Mar 2023 16:43:34 -0300 Subject: [PATCH 58/88] feat(kafka_consumer): add `offset_commit_interval_seconds` kafka parameter --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 10 ++++++++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 5 +++++ .../src/kafka/emqx_bridge_impl_kafka_consumer.erl | 8 +++++++- .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 53bc5dddd..c6bc52bdd 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -597,4 +597,14 @@ emqx_ee_bridge_kafka { zh: "偏移重置政策" } } + consumer_offset_commit_interval_seconds { + desc { + en: "Defines the time interval between two OffsetCommitRequest messages." + zh: "定义了两个OffsetCommitRequest消息之间的时间间隔。" + } + label { + en: "Offset Commit Interval" + zh: "偏移承诺间隔" + } + } } diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 865a5f64b..89cad3421 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -280,6 +280,11 @@ fields(consumer_kafka_opts) -> mk( enum([reset_to_latest, reset_to_earliest, reset_by_subscriber]), #{default => reset_to_latest, desc => ?DESC(consumer_offset_reset_policy)} + )}, + {offset_commit_interval_seconds, + mk( + pos_integer(), + #{default => 5, desc => ?DESC(consumer_offset_commit_interval_seconds)} )} ]. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index 076d7fd97..a38807f91 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -37,6 +37,7 @@ kafka := #{ max_batch_bytes := emqx_schema:bytesize(), max_rejoin_attempts := non_neg_integer(), + offset_commit_interval_seconds := pos_integer(), offset_reset_policy := offset_reset_policy(), topic := binary() }, @@ -90,6 +91,7 @@ on_start(InstanceId, Config) -> kafka := #{ max_batch_bytes := _, max_rejoin_attempts := _, + offset_commit_interval_seconds := _, offset_reset_policy := _, topic := _ }, @@ -248,6 +250,7 @@ start_subscriber(Config, InstanceId, ClientID) -> kafka := #{ max_batch_bytes := MaxBatchBytes, max_rejoin_attempts := MaxRejoinAttempts, + offset_commit_interval_seconds := OffsetCommitInterval, offset_reset_policy := OffsetResetPolicy, topic := KafkaTopic }, @@ -272,7 +275,10 @@ start_subscriber(Config, InstanceId, ClientID) -> {max_bytes, MaxBatchBytes}, {offset_reset_policy, OffsetResetPolicy} ], - GroupConfig = [{max_rejoin_attempts, MaxRejoinAttempts}], + GroupConfig = [ + {max_rejoin_attempts, MaxRejoinAttempts}, + {offset_commit_interval_seconds, OffsetCommitInterval} + ], GroupSubscriberConfig = #{ client => ClientID, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index ce5003a8b..086699a07 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -494,6 +494,7 @@ kafka_config(TestCase, _KafkaType, Config) -> " topic = ~s\n" " max_batch_bytes = 896KB\n" " max_rejoin_attempts = 5\n" + " offset_commit_interval_seconds = 3\n" %% todo: matrix this " offset_reset_policy = reset_to_latest\n" " }\n" From 1f31a87974f1d76d355f5c26ab6e365420d9964a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 7 Mar 2023 16:42:42 -0300 Subject: [PATCH 59/88] fix(bridge): improve macro var usage Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index f29115b0a..7b879132c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -46,10 +46,10 @@ %% bi-directional bridge with producer/consumer or ingress/egress configs -define(IS_BI_DIR_BRIDGE(TYPE), - TYPE =:= <<"mqtt">> + (TYPE) =:= <<"mqtt">> ). -define(IS_INGRESS_BRIDGE(TYPE), - TYPE =:= <<"kafka_consumer">> orelse ?IS_BI_DIR_BRIDGE(TYPE) + (TYPE) =:= <<"kafka_consumer">> orelse ?IS_BI_DIR_BRIDGE(TYPE) ). -if(?EMQX_RELEASE_EDITION == ee). From 1b78f22d921b023c6e38c8cd561cad0d7557b38f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 7 Mar 2023 17:38:46 -0300 Subject: [PATCH 60/88] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- changes/ee/feat-9564.zh.md | 2 +- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/changes/ee/feat-9564.zh.md b/changes/ee/feat-9564.zh.md index 8eb29998b..0c4867e46 100644 --- a/changes/ee/feat-9564.zh.md +++ b/changes/ee/feat-9564.zh.md @@ -1,2 +1,2 @@ -实现了Kafka消费者桥。 +实现了 Kafka 消费者桥接。 现在可以从Kafka消费消息并将其发布到 MQTT主题。 diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index c6bc52bdd..6b770a752 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -51,8 +51,8 @@ emqx_ee_bridge_kafka { } producer_opts { desc { - en: "Local MQTT data source and Kafka bridge configs." - zh: "本地 MQTT 数据源和 Kafka 桥接的配置。" + en: "Local MQTT data source and Kafka bridge configs. Should not configure this if the bridge is used as a rule action." + zh: "本地 MQTT 数据源和 Kafka 桥接的配置。若该桥接用于规则的动作，则必须将该配置项删除。" } label { en: "MQTT to Kafka" @@ -477,7 +477,7 @@ emqx_ee_bridge_kafka { kafka_consumer { desc { en: "Kafka Consumer configuration." - zh: "Kafka Consumer的配置。" + zh: "Kafka 消费者配置。" } label { en: "Kafka Consumer" From 9037314aea63ef5c70ab6c63aa7908976f9383a1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Mar 2023 09:29:05 -0300 Subject: [PATCH 61/88] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- .../i18n/emqx_ee_bridge_kafka.conf | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 6b770a752..6d2ca46ab 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -481,13 +481,13 @@ emqx_ee_bridge_kafka { } label { en: "Kafka Consumer" - zh: "Kafka Consumer" + zh: "Kafka 消费者" } } consumer_opts { desc { - en: "Local MQTT data sink and Kafka bridge configs." - zh: "本地MQTT数据汇和Kafka桥配置。" + en: "Local MQTT publish and Kafka consumer configs." + zh: "本地 MQTT 转发 和 Kafka 消费者配置。" } label { en: "MQTT to Kafka" @@ -501,23 +501,23 @@ emqx_ee_bridge_kafka { } label { en: "Kafka Consumer" - zh: "卡夫卡消费者" + zh: "Kafka 消费者" } } consumer_mqtt_opts { desc { - en: "MQTT data sink." - zh: "MQTT数据汇。" + en: "Local MQTT message publish." + zh: "本地 MQTT 消息转发。" } label { - en: "MQTT data sink." - zh: "MQTT数据汇。" + en: "MQTT publish" + zh: "MQTT 转发" } } consumer_mqtt_topic { desc { en: "Local topic to which consumed Kafka messages should be published to." - zh: "消耗的Kafka消息应该被发布到的本地主题。" + zh: "设置 Kafka 消息向哪个本地 MQTT 主题转发消息。" } label { en: "MQTT Topic" @@ -527,48 +527,44 @@ emqx_ee_bridge_kafka { consumer_mqtt_qos { desc { en: "MQTT QoS used to publish messages consumed from Kafka." - zh: "MQTT QoS用于发布从Kafka消耗的消息。" + zh: "转发 MQTT 消息时使用的 QoS。" } label { - en: "MQTT Topic QoS" - zh: "MQTT 主题服务质量" + en: "QoS" + zh: "QoS" } } consumer_mqtt_payload { desc { en: "The payload of the MQTT message to be published.\n" - "full_message will encode all data available as a JSON object," + "full_message will encode available Kafka message attributes as a JSON object, including Key, Value, Timestamp and Headers" "message_value will directly use the Kafka message value as the " "MQTT message payload." zh: "要发布的MQTT消息的有效载荷。" - "full_message将把所有可用数据编码为JSON对象，" - "message_value将直接使用Kafka消息值作为MQTT消息的有效载荷。" + "full_message将把所有可用数据编码为JSON对象，包括 Key，Value，Timestamp 和 Headers。" + "message_value将直接使用 Kafka 消息值作为MQTT消息的 Payload。" } label { en: "MQTT Payload" - zh: "MQTT有效载荷" + zh: "MQTT Payload" } } consumer_kafka_topic { desc { en: "Kafka topic to consume from." - zh: "从Kafka主题消费。" + zh: "指定从哪个 Kafka 主题消费消息。" } label { en: "Kafka topic" - zh: "卡夫卡主题 " + zh: "Kafka 主题 " } } consumer_max_batch_bytes { desc { en: "Maximum bytes to fetch in a batch of messages." - "NOTE: this value might be expanded to retry when " - "it is not enough to fetch even a single message, " - "then slowly shrink back to the given value." + "Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance." zh: "在一批消息中要取的最大字节数。" - "注意：这个值可能会被扩大，" - "当它甚至不足以取到一条消息时，就会重试，" - "然后慢慢缩回到给定的值。" + "如果该配置小于 Kafka 中消息到大小，则可能会影响消费性能。" } label { en: "Max Bytes" @@ -594,13 +590,13 @@ emqx_ee_bridge_kafka { } label { en: "Offset Reset Policy" - zh: "偏移重置政策" + zh: "偏移重置策略" } } consumer_offset_commit_interval_seconds { desc { - en: "Defines the time interval between two OffsetCommitRequest messages." - zh: "定义了两个OffsetCommitRequest消息之间的时间间隔。" + en: "Defines the time interval between two offset commit requests sent for each consumer group." + zh: "指定 Kafka 消费组偏移量提交的时间间隔。" } label { en: "Offset Commit Interval" From 2a3aef92bb9cf0a8dc46522a79c10b4614a779e4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Mar 2023 09:35:17 -0300 Subject: [PATCH 62/88] refactor(kafka_consumer): redact error info --- .../src/kafka/emqx_bridge_impl_kafka_consumer.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index a38807f91..ab4a996bd 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -125,7 +125,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_kafka_consumer_client", instance_id => InstanceId, kafka_hosts => BootstrapHosts, - reason => Reason + reason => emqx_misc:redact(Reason) }), throw(failed_to_start_kafka_client) end, @@ -312,7 +312,7 @@ start_subscriber(Config, InstanceId, ClientID) -> instance_id => InstanceId, kafka_hosts => emqx_bridge_impl_kafka:hosts(BootstrapHosts0), kafka_topic => KafkaTopic, - reason => Reason2 + reason => emqx_misc:redact(Reason2) }), stop_client(ClientID), throw(failed_to_start_kafka_consumer) From fecc08bb8f0a2e35564dac9cc4c7b47aa24674d2 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Mar 2023 09:37:48 -0300 Subject: [PATCH 63/88] refactor(kafka_consumer): apply review suggestions --- .../src/emqx_ee_bridge_kafka.erl | 4 +- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 81 ++++++++++--------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 89cad3421..f812901be 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -274,7 +274,9 @@ fields(consumer_kafka_opts) -> })}, {max_rejoin_attempts, mk(non_neg_integer(), #{ - default => 5, desc => ?DESC(consumer_max_rejoin_attempts) + hidden => true, + default => 5, + desc => ?DESC(consumer_max_rejoin_attempts) })}, {offset_reset_policy, mk( diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index ab4a996bd..f89b63d7b 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -129,7 +129,7 @@ on_start(InstanceId, Config) -> }), throw(failed_to_start_kafka_client) end, - start_subscriber(Config, InstanceId, ClientID). + start_consumer(Config, InstanceId, ClientID). -spec on_stop(manager_id(), state()) -> ok. on_stop(_InstanceID, State) -> @@ -169,44 +169,45 @@ handle_message(Message, State) -> ?tp_span( kafka_consumer_handle_message, #{message => Message, state => State}, - begin - #{ - resource_id := ResourceId, - hookpoint := Hookpoint, - kafka_topic := KafkaTopic, - mqtt := #{ - topic := MQTTTopic, - payload := MQTTPayload, - qos := MQTTQoS - } - } = State, - FullMessage = #{ - offset => Message#kafka_message.offset, - key => Message#kafka_message.key, - value => Message#kafka_message.value, - ts => Message#kafka_message.ts, - ts_type => Message#kafka_message.ts_type, - headers => maps:from_list(Message#kafka_message.headers), - topic => KafkaTopic - }, - Payload = - case MQTTPayload of - full_message -> - FullMessage; - message_value -> - Message#kafka_message.value - end, - EncodedPayload = emqx_json:encode(Payload), - MQTTMessage = emqx_message:make(ResourceId, MQTTQoS, MQTTTopic, EncodedPayload), - _ = emqx:publish(MQTTMessage), - emqx:run_hook(Hookpoint, [FullMessage]), - emqx_resource_metrics:received_inc(ResourceId), - %% note: just `ack' does not commit the offset to the - %% kafka consumer group. - {ok, commit, State} - end + do_handle_message(Message, State) ). +do_handle_message(Message, State) -> + #{ + resource_id := ResourceId, + hookpoint := Hookpoint, + kafka_topic := KafkaTopic, + mqtt := #{ + topic := MQTTTopic, + payload := MQTTPayload, + qos := MQTTQoS + } + } = State, + FullMessage = #{ + offset => Message#kafka_message.offset, + key => Message#kafka_message.key, + value => Message#kafka_message.value, + ts => Message#kafka_message.ts, + ts_type => Message#kafka_message.ts_type, + headers => maps:from_list(Message#kafka_message.headers), + topic => KafkaTopic + }, + Payload = + case MQTTPayload of + full_message -> + FullMessage; + message_value -> + Message#kafka_message.value + end, + EncodedPayload = emqx_json:encode(Payload), + MQTTMessage = emqx_message:make(ResourceId, MQTTQoS, MQTTTopic, EncodedPayload), + _ = emqx:publish(MQTTMessage), + emqx:run_hook(Hookpoint, [FullMessage]), + emqx_resource_metrics:received_inc(ResourceId), + %% note: just `ack' does not commit the offset to the + %% kafka consumer group. + {ok, commit, State}. + %%------------------------------------------------------------------------------------- %% Helper fns %%------------------------------------------------------------------------------------- @@ -241,8 +242,8 @@ ensure_consumer_supervisor_started() -> ok end. --spec start_subscriber(config(), manager_id(), brod:client_id()) -> {ok, state()}. -start_subscriber(Config, InstanceId, ClientID) -> +-spec start_consumer(config(), manager_id(), brod:client_id()) -> {ok, state()}. +start_consumer(Config, InstanceId, ClientID) -> #{ bootstrap_hosts := BootstrapHosts0, bridge_name := BridgeName, @@ -256,7 +257,7 @@ start_subscriber(Config, InstanceId, ClientID) -> }, mqtt := #{topic := MQTTTopic, qos := MQTTQoS, payload := MQTTPayload} } = Config, - ensure_consumer_supervisor_started(), + ok = ensure_consumer_supervisor_started(), InitialState = #{ resource_id => emqx_bridge_resource:resource_id(kafka_consumer, BridgeName), mqtt => #{ From 27dfd98f46ed7973d40928b2e72079300c747412 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Mar 2023 11:29:12 -0300 Subject: [PATCH 64/88] fix(kafka): fix api methods schemas and bridge type enum --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 7 ++-- .../src/emqx_ee_bridge_kafka.erl | 39 +++++++++++++------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 2d7f5b5be..0746736f3 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -15,7 +15,8 @@ api_schemas(Method) -> [ ref(emqx_ee_bridge_gcp_pubsub, Method), - ref(emqx_ee_bridge_kafka, Method), + ref(emqx_ee_bridge_kafka, Method ++ "_consumer"), + ref(emqx_ee_bridge_kafka, Method ++ "_producer"), ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), @@ -147,9 +148,9 @@ kafka_structs() -> {Type, mk( hoconsc:map(name, ref(emqx_ee_bridge_kafka, Type)), - #{desc => <<"EMQX Enterprise Config">>, required => false} + #{desc => <<"Kafka ", Name/binary, " Bridge Config">>, required => false} )} - || Type <- [kafka_producer, kafka_consumer] + || {Type, Name} <- [{kafka_producer, <<"Producer">>}, {kafka_consumer, <<"Consumer">>}] ]. influxdb_structs() -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index f812901be..d8f7f7fc8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -36,8 +36,14 @@ conn_bridge_examples(Method) -> [ #{ - <<"kafka">> => #{ - summary => <<"Kafka Bridge">>, + <<"kafka_producer">> => #{ + summary => <<"Kafka Producer Bridge">>, + value => values(Method) + } + }, + #{ + <<"kafka_consumer">> => #{ + summary => <<"Kafka Consumer Bridge">>, value => values(Method) } } @@ -60,14 +66,18 @@ host_opts() -> namespace() -> "bridge_kafka". -roots() -> ["config"]. +roots() -> ["config_consumer", "config_producer"]. -fields("post") -> - [type_field(), name_field() | fields("config")]; -fields("put") -> - fields("config"); -fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"); +fields("post_" ++ Type) -> + [type_field(), name_field() | fields("config_" ++ Type)]; +fields("put_" ++ Type) -> + fields("config_" ++ Type); +fields("get_" ++ Type) -> + emqx_bridge_schema:status_fields() ++ fields("post_" ++ Type); +fields("config_producer") -> + fields(kafka_producer); +fields("config_consumer") -> + fields(kafka_consumer); fields(kafka_producer) -> fields("config") ++ fields(producer_opts); fields(kafka_consumer) -> @@ -292,8 +302,12 @@ fields(consumer_kafka_opts) -> desc("config") -> ?DESC("desc_config"); -desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> - ["Configuration for Kafka using `", string:to_upper(Method), "` method."]; +desc("get_" ++ Type) when Type =:= "consumer"; Type =:= "producer" -> + ["Configuration for Kafka using `GET` method."]; +desc("put_" ++ Type) when Type =:= "consumer"; Type =:= "producer" -> + ["Configuration for Kafka using `PUT` method."]; +desc("post_" ++ Type) when Type =:= "consumer"; Type =:= "producer" -> + ["Configuration for Kafka using `POST` method."]; desc(Name) -> lists:member(Name, struct_names()) orelse throw({missing_desc, Name}), ?DESC(Name). @@ -317,7 +331,8 @@ struct_names() -> %% ------------------------------------------------------------------------------------------------- %% internal type_field() -> - {type, mk(enum([kafka]), #{required => true, desc => ?DESC("desc_type")})}. + {type, + mk(enum([kafka_consumer, kafka_producer]), #{required => true, desc => ?DESC("desc_type")})}. name_field() -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. From 03342923b993ddcc655b786d470e4ff114ddee87 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Mar 2023 11:29:46 -0300 Subject: [PATCH 65/88] fix(bridge): use the same dry run prefix Kafka Producer and Consumer bridges rely on this prefix for detecting a dry run and avoid leaking atoms. At some point, this prefix was changed, effectively disabling the check in Kafka Producer. --- apps/emqx_bridge/rebar.config | 4 +++- apps/emqx_bridge/src/emqx_bridge_resource.erl | 3 ++- apps/emqx_resource/include/emqx_resource.hrl | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 0a1cbc29b..79f2caf50 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,5 +1,7 @@ {erl_opts, [debug_info]}. -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ {emqx, {path, "../emqx"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 7b879132c..fde823ea7 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -16,6 +16,7 @@ -module(emqx_bridge_resource). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). -export([ bridge_to_resource_type/1, @@ -224,7 +225,7 @@ recreate(Type, Name, Conf, Opts) -> ). create_dry_run(Type, Conf0) -> - TmpPath0 = iolist_to_binary(["bridges-create-dry-run:", emqx_misc:gen_id(8)]), + TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_misc:gen_id(8)]), TmpPath = emqx_misc:safe_filename(TmpPath0), Conf = emqx_map_lib:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index be570e694..283156b95 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -119,5 +119,5 @@ -define(AUTO_RESTART_INTERVAL, 60000). -define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>). --define(TEST_ID_PREFIX, "_test_:"). +-define(TEST_ID_PREFIX, "_test-create-dry-run:"). -define(RES_METRICS, resource_metrics). From c182a4053e85027ec63a6b8f1264cebe0a98591a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 8 Mar 2023 11:30:58 -0300 Subject: [PATCH 66/88] fix(kafka_consumer): avoid leaking atoms in bridge probe API --- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 36 +++++++++++++++---- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 29 ++++++++++++++- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index f89b63d7b..43717dd89 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -101,8 +101,7 @@ on_start(InstanceId, Config) -> BootstrapHosts = emqx_bridge_impl_kafka:hosts(BootstrapHosts0), KafkaType = kafka_consumer, %% Note: this is distinct per node. - ClientID0 = emqx_bridge_impl_kafka:make_client_id(KafkaType, BridgeName), - ClientID = binary_to_atom(ClientID0), + ClientID = make_client_id(InstanceId, KafkaType, BridgeName), ClientOpts0 = case Auth of none -> []; @@ -217,9 +216,9 @@ add_ssl_opts(ClientOpts, #{enable := false}) -> add_ssl_opts(ClientOpts, SSL) -> [{ssl, emqx_tls_lib:to_client_opts(SSL)} | ClientOpts]. --spec make_subscriber_id(atom()) -> emqx_ee_bridge_kafka_consumer_sup:child_id(). +-spec make_subscriber_id(atom() | binary()) -> emqx_ee_bridge_kafka_consumer_sup:child_id(). make_subscriber_id(BridgeName) -> - BridgeNameBin = atom_to_binary(BridgeName), + BridgeNameBin = to_bin(BridgeName), <<"kafka_subscriber:", BridgeNameBin/binary>>. ensure_consumer_supervisor_started() -> @@ -398,7 +397,32 @@ log_when_error(Fun, Log) -> }) end. --spec consumer_group_id(atom()) -> binary(). +-spec consumer_group_id(atom() | binary()) -> binary(). consumer_group_id(BridgeName0) -> - BridgeName = atom_to_binary(BridgeName0), + BridgeName = to_bin(BridgeName0), <<"emqx-kafka-consumer:", BridgeName/binary>>. + +-spec is_dry_run(manager_id()) -> boolean(). +is_dry_run(InstanceId) -> + TestIdStart = string:find(InstanceId, ?TEST_ID_PREFIX), + case TestIdStart of + nomatch -> + false; + _ -> + string:equal(TestIdStart, InstanceId) + end. + +-spec make_client_id(manager_id(), kafka_consumer, atom() | binary()) -> atom(). +make_client_id(InstanceId, KafkaType, KafkaName) -> + case is_dry_run(InstanceId) of + false -> + ClientID0 = emqx_bridge_impl_kafka:make_client_id(KafkaType, KafkaName), + binary_to_atom(ClientID0); + true -> + %% It is a dry run and we don't want to leak too many + %% atoms. + probing_brod_consumers + end. + +to_bin(B) when is_binary(B) -> B; +to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 086699a07..e0be06e1a 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -594,12 +594,29 @@ update_bridge_api(Config, Overrides) -> ct:pal("updating bridge (via http): ~p", [Params]), Res = case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of - {ok, Res0} -> {ok, emqx_json:decode(Res0, [return_maps])}; + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_json:decode(Body0, [return_maps])}; Error -> Error end, ct:pal("bridge update result: ~p", [Res]), Res. +probe_bridge_api(Config) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(kafka_name, Config), + KafkaConfig = ?config(kafka_config, Config), + Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("probing bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; + Error -> Error + end, + ct:pal("bridge probe result: ~p", [Res]), + Res. + send_message(Config, Payload) -> Name = ?config(kafka_name, Config), Type = ?BRIDGE_TYPE_BIN, @@ -866,6 +883,16 @@ t_start_and_consume_ok(Config) -> #{?snk_kind := kafka_consumer_handle_message, ?snk_span := {complete, _}}, 20_000 ), + + %% Check that the bridge probe API doesn't leak atoms. + ProbeRes = probe_bridge_api(Config), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + AtomsBefore = erlang:system_info(atom_count), + %% Probe again; shouldn't have created more atoms. + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + AtomsAfter = erlang:system_info(atom_count), + ?assertEqual(AtomsBefore, AtomsAfter), + Res end, fun({_Partition, OffsetReply}, Trace) -> From 5eaaa83b820ea31b4658ec3f9d16bd55f2cdccab Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 8 Mar 2023 18:53:08 +0100 Subject: [PATCH 67/88] chore: simplify run.sh - get rid of sudo - do not change permissions of existing files - use our own docker image to generate certs to make it working on arm - bump kafka docker image version to have access to multiplatofrm one --- .../docker-compose-kafka.yaml | 9 ++-- .ci/docker-compose-file/docker-compose.yaml | 2 +- .../kafka/generate-certs.sh | 46 ------------------- .../kafka/kafka-entrypoint.sh | 1 + scripts/ct/run.sh | 38 +++------------ 5 files changed, 12 insertions(+), 84 deletions(-) delete mode 100755 .ci/docker-compose-file/kafka/generate-certs.sh diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index e54f1377d..d4989bd0b 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -10,13 +10,12 @@ services: networks: emqx_bridge: ssl_cert_gen: - image: fredrikhgrelland/alpine-jdk11-openssl + # see https://github.com/emqx/docker-images + image: ghcr.io/emqx/certgen:latest container_name: ssl_cert_gen + user: "${DOCKER_USER:-root}" volumes: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - - ./kafka/generate-certs.sh:/bin/generate-certs.sh - entrypoint: /bin/sh - command: /bin/generate-certs.sh kdc: hostname: kdc.emqx.net image: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04 @@ -36,7 +35,7 @@ services: - ./kerberos/run.sh:/usr/bin/run.sh command: run.sh kafka_1: - image: wurstmeister/kafka:2.13-2.7.0 + image: wurstmeister/kafka:2.13-2.8.1 # ports: # - "9192-9195:9192-9195" container_name: kafka-1.emqx.net diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 42003fcb7..5c37d971e 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -23,7 +23,7 @@ services: - ./kerberos/krb5.conf:/etc/krb5.conf working_dir: /emqx tty: true - user: "${UID_GID}" + user: "${DOCKER_USER:-root}" networks: emqx_bridge: diff --git a/.ci/docker-compose-file/kafka/generate-certs.sh b/.ci/docker-compose-file/kafka/generate-certs.sh deleted file mode 100755 index 3f1c75550..000000000 --- a/.ci/docker-compose-file/kafka/generate-certs.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/bash - -set -euo pipefail - -set -x - -# Source https://github.com/zmstone/docker-kafka/blob/master/generate-certs.sh - -HOST="*." -DAYS=3650 -PASS="password" - -cd /var/lib/secret/ - -# Delete old files -(rm ca.key ca.crt server.key server.csr server.crt client.key client.csr client.crt server.p12 kafka.keystore.jks kafka.truststore.jks 2>/dev/null || true) - -ls - -echo '== Generate self-signed server and client certificates' -echo '= generate CA' -openssl req -new -x509 -keyout ca.key -out ca.crt -days $DAYS -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST" - -echo '= generate server certificate request' -openssl req -newkey rsa:2048 -sha256 -keyout server.key -out server.csr -days "$DAYS" -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST" - -echo '= sign server certificate' -openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days "$DAYS" -CAcreateserial - -echo '= generate client certificate request' -openssl req -newkey rsa:2048 -sha256 -keyout client.key -out client.csr -days "$DAYS" -nodes -subj "/C=SE/ST=Stockholm/L=Stockholm/O=brod/OU=test/CN=$HOST" - -echo '== sign client certificate' -openssl x509 -req -CA ca.crt -CAkey ca.key -in client.csr -out client.crt -days $DAYS -CAserial ca.srl - -echo '= Convert self-signed certificate to PKCS#12 format' -openssl pkcs12 -export -name "$HOST" -in server.crt -inkey server.key -out server.p12 -CAfile ca.crt -passout pass:"$PASS" - -echo '= Import PKCS#12 into a java keystore' - -echo $PASS | keytool -importkeystore -destkeystore kafka.keystore.jks -srckeystore server.p12 -srcstoretype pkcs12 -alias "$HOST" -storepass "$PASS" - - -echo '= Import CA into java truststore' - -echo yes | keytool -keystore kafka.truststore.jks -alias CARoot -import -file ca.crt -storepass "$PASS" diff --git a/.ci/docker-compose-file/kafka/kafka-entrypoint.sh b/.ci/docker-compose-file/kafka/kafka-entrypoint.sh index 445fd65c9..336a78e74 100755 --- a/.ci/docker-compose-file/kafka/kafka-entrypoint.sh +++ b/.ci/docker-compose-file/kafka/kafka-entrypoint.sh @@ -17,6 +17,7 @@ timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.keytab ]; do sleep 1; echo "+++++++ Wait until SSL certs are generated ++++++++" timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.truststore.jks ]; do sleep 1; done' +keytool -list -v -keystore /var/lib/secret/kafka.keystore.jks -storepass password sleep 3 diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index b3c424ea1..164f38ba3 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -154,9 +154,6 @@ for dep in ${CT_DEPS}; do '.ci/docker-compose-file/docker-compose-pgsql-tls.yaml' ) ;; kafka) - # Kafka container generates root owned ssl files - # the files are shared with EMQX (with a docker volume) - NEED_ROOT=yes FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' ) ;; tdengine) @@ -180,35 +177,14 @@ F_OPTIONS="" for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done -ORIG_UID_GID="$UID:$UID" -if [[ "${NEED_ROOT:-}" == 'yes' ]]; then - export UID_GID='root:root' -else - # Passing $UID to docker-compose to be used in erlang container - # as owner of the main process to avoid git repo permissions issue. - # Permissions issue happens because we are mounting local filesystem - # where files are owned by $UID to docker container where it's using - # root (UID=0) by default, and git is not happy about it. - export UID_GID="$ORIG_UID_GID" -fi -# /emqx is where the source dir is mounted to the Erlang container -# in .ci/docker-compose-file/docker-compose.yaml +export DOCKER_USER="$(id -u)" + TTY='' if [[ -t 1 ]]; then TTY='-t' fi -function restore_ownership { - if [[ -n ${EMQX_TEST_DO_NOT_RUN_SUDO+x} ]] || ! sudo chown -R "$ORIG_UID_GID" . >/dev/null 2>&1; then - docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $ORIG_UID_GID /emqx" >/dev/null 2>&1 || true - fi -} - -restore_ownership -trap restore_ownership EXIT - - if [ "$STOP" = 'no' ]; then # some left-over log file has to be deleted before a new docker-compose up rm -f '.ci/docker-compose-file/redis/*.log' @@ -216,11 +192,10 @@ if [ "$STOP" = 'no' ]; then $DC $F_OPTIONS up -d --build --remove-orphans fi -echo "Fixing file owners and permissions for $UID_GID" -# rebar and hex cache directory need to be writable by $UID -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $UID_GID /.cache && chown -R $UID_GID /emqx/.git /emqx/.ci /emqx/_build/default/lib" -# need to initialize .erlang.cookie manually here because / is not writable by $UID -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie" +# rebar and hex cache directory need to be writable by $DOCKER_USER +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $DOCKER_USER /.cache" +# need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie" if [ "$ONLY_UP" = 'yes' ]; then exit 0 @@ -242,7 +217,6 @@ else docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "./rebar3 ct $REBAR3CT" fi RESULT=$? - restore_ownership if [ $RESULT -ne 0 ]; then LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" From 2fe341d152c9c441a0dbc1b4046974b9af9158f2 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 9 Mar 2023 09:12:46 +0100 Subject: [PATCH 68/88] chore(run.sh): fix permissions on secrets directory --- scripts/ct/run.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 164f38ba3..e4882695e 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -185,11 +185,25 @@ if [[ -t 1 ]]; then TTY='-t' fi +# ensure directory with secrets is created by current user before running compose +mkdir -p /tmp/emqx-ci/emqx-shared-secret + if [ "$STOP" = 'no' ]; then # some left-over log file has to be deleted before a new docker-compose up rm -f '.ci/docker-compose-file/redis/*.log' + set +e # shellcheck disable=2086 # no quotes for F_OPTIONS $DC $F_OPTIONS up -d --build --remove-orphans + RESULT=$? + if [ $RESULT -ne 0 ]; then + mkdir -p _build/test/logs + LOG='_build/test/logs/docker-compose.log' + echo "Dumping docker-compose log to $LOG" + # shellcheck disable=2086 # no quotes for F_OPTIONS + $DC $F_OPTIONS logs --no-color --timestamps > "$LOG" + exit 1 + fi + set -e fi # rebar and hex cache directory need to be writable by $DOCKER_USER From a59827cc6acfc0076aa5473893425192bc522fee Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 9 Mar 2023 12:07:48 +0100 Subject: [PATCH 69/88] chore(run.sh): prefer docker compose plugin over docker-compose --- scripts/ct/run.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index e4882695e..17319dc51 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -21,11 +21,16 @@ help() { echo " otherwise it runs the entire app's CT" } -if command -v docker-compose; then +set +e +if docker compose version; then + DC='docker compose' +elif command -v docker-compose; then DC='docker-compose' else - DC='docker compose' + echo 'Neither "docker compose" or "docker-compose" are available, stop.' + exit 1 fi +set -e WHICH_APP='novalue' CONSOLE='no' From 9d6af17f697cd9bd5243876e354ceb319543d517 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 9 Mar 2023 10:44:47 -0300 Subject: [PATCH 70/88] ci(fix): create and give permissions to mix directories --- scripts/ct/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 17319dc51..279b979a3 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -211,8 +211,8 @@ if [ "$STOP" = 'no' ]; then set -e fi -# rebar and hex cache directory need to be writable by $DOCKER_USER -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $DOCKER_USER /.cache" +# rebar, mix and hex cache directory need to be writable by $DOCKER_USER +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache /.hex /.mix && chown $DOCKER_USER /.cache /.hex /.mix" # need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie" From e4f058ce2a55a6c9c14e45835732eee6cfa9bb04 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 9 Mar 2023 10:55:32 -0300 Subject: [PATCH 71/88] ci(fix): create user inside container so `emqx console` works --- scripts/ct/run.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 279b979a3..b3ae29759 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -215,6 +215,9 @@ fi docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache /.hex /.mix && chown $DOCKER_USER /.cache /.hex /.mix" # need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie" +# the user must exist inside the container for `whoami` to work +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "useradd --uid $DOCKER_USER -M -d / emqx" || true +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $DOCKER_USER /var/lib/secret" || true if [ "$ONLY_UP" = 'yes' ]; then exit 0 @@ -236,7 +239,7 @@ else docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "./rebar3 ct $REBAR3CT" fi RESULT=$? - if [ $RESULT -ne 0 ]; then + if [ "$RESULT" -ne 0 ]; then LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" # shellcheck disable=2086 # no quotes for F_OPTIONS @@ -246,5 +249,5 @@ else # shellcheck disable=2086 # no quotes for F_OPTIONS $DC $F_OPTIONS down fi - exit $RESULT + exit "$RESULT" fi From 8b1fa50413e6c8f3846dd752846778adb8859237 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 9 Mar 2023 15:39:40 -0300 Subject: [PATCH 72/88] ci(fix): fix shellsheck warning --- scripts/ct/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index b3ae29759..bf7b2073d 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -163,7 +163,7 @@ for dep in ${CT_DEPS}; do ;; tdengine) FILES+=( '.ci/docker-compose-file/docker-compose-tdengine-restful.yaml' ) - ;; + ;; clickhouse) FILES+=( '.ci/docker-compose-file/docker-compose-clickhouse.yaml' ) ;; @@ -183,7 +183,8 @@ for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done -export DOCKER_USER="$(id -u)" +DOCKER_USER="$(id -u)" +export DOCKER_USER TTY='' if [[ -t 1 ]]; then From f31f15e44ee5dec1b0666c37690447c50816e860 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 9 Mar 2023 15:06:16 -0300 Subject: [PATCH 73/88] chore(kafka_producer): make schema changes more backwards compatible This will still require fixes to the frontend. --- apps/emqx_bridge/src/emqx_bridge.erl | 5 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 +- .../i18n/emqx_ee_bridge_kafka.conf | 8 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 23 ++- .../src/emqx_ee_bridge_kafka.erl | 34 +++- .../kafka/emqx_bridge_impl_kafka_producer.erl | 11 +- .../emqx_bridge_impl_kafka_producer_SUITE.erl | 34 ++-- .../test/emqx_ee_bridge_kafka_tests.erl | 181 ++++++++++++++++++ 8 files changed, 270 insertions(+), 30 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 5bc83dbd9..d5d5adff1 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -55,7 +55,10 @@ T == gcp_pubsub; T == influxdb_api_v1; T == influxdb_api_v2; - T == kafka_producer; + %% TODO: rename this to `kafka_producer' after alias support is + %% added to hocon; keeping this as just `kafka' for backwards + %% compatibility. + T == kafka; T == redis_single; T == redis_sentinel; T == redis_cluster; diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index fde823ea7..6426a46b7 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -309,7 +309,9 @@ parse_confs(Type, Name, Conf) when ?IS_INGRESS_BRIDGE(Type) -> %% receives a message from the external database. BId = bridge_id(Type, Name), Conf#{hookpoint => <<"$bridges/", BId/binary>>, bridge_name => Name}; -parse_confs(<<"kafka_producer">> = _Type, Name, Conf) -> +%% TODO: rename this to `kafka_producer' after alias support is added +%% to hocon; keeping this as just `kafka' for backwards compatibility. +parse_confs(<<"kafka">> = _Type, Name, Conf) -> Conf#{bridge_name => Name}; parse_confs(_Type, _Name, Conf) -> Conf. diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 6d2ca46ab..636573d07 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -51,8 +51,8 @@ emqx_ee_bridge_kafka { } producer_opts { desc { - en: "Local MQTT data source and Kafka bridge configs. Should not configure this if the bridge is used as a rule action." - zh: "本地 MQTT 数据源和 Kafka 桥接的配置。若该桥接用于规则的动作，则必须将该配置项删除。" + en: "Local MQTT data source and Kafka bridge configs." + zh: "本地 MQTT 数据源和 Kafka 桥接的配置。" } label { en: "MQTT to Kafka" @@ -61,8 +61,8 @@ emqx_ee_bridge_kafka { } mqtt_topic { desc { - en: "MQTT topic or topic as data source (bridge input)." - zh: "指定 MQTT 主题作为桥接的数据源" + en: "MQTT topic or topic as data source (bridge input). Should not configure this if the bridge is used as a rule action." + zh: "指定 MQTT 主题作为桥接的数据源。 若该桥接用于规则的动作，则必须将该配置项删除。" } label { en: "Source MQTT Topic" diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 0746736f3..ec81b7935 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -66,7 +66,9 @@ examples(Method) -> resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); resource_type(kafka_consumer) -> emqx_bridge_impl_kafka_consumer; -resource_type(kafka_producer) -> emqx_bridge_impl_kafka_producer; +%% TODO: rename this to `kafka_producer' after alias support is added +%% to hocon; keeping this as just `kafka' for backwards compatibility. +resource_type(kafka) -> emqx_bridge_impl_kafka_producer; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; @@ -145,12 +147,23 @@ mongodb_structs() -> kafka_structs() -> [ - {Type, + %% TODO: rename this to `kafka_producer' after alias support + %% is added to hocon; keeping this as just `kafka' for + %% backwards compatibility. + {kafka, mk( - hoconsc:map(name, ref(emqx_ee_bridge_kafka, Type)), - #{desc => <<"Kafka ", Name/binary, " Bridge Config">>, required => false} + hoconsc:map(name, ref(emqx_ee_bridge_kafka, kafka_producer)), + #{ + desc => <<"Kafka Producer Bridge Config">>, + required => false, + converter => fun emqx_ee_bridge_kafka:kafka_producer_converter/2 + } + )}, + {kafka_consumer, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_kafka, kafka_consumer)), + #{desc => <<"Kafka Consumer Bridge Config">>, required => false} )} - || {Type, Name} <- [{kafka_producer, <<"Producer">>}, {kafka_consumer, <<"Consumer">>}] ]. influxdb_structs() -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index d8f7f7fc8..583acc48d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -30,13 +30,18 @@ host_opts/0 ]). +-export([kafka_producer_converter/2]). + %% ------------------------------------------------------------------------------------------------- %% api conn_bridge_examples(Method) -> [ #{ - <<"kafka_producer">> => #{ + %% TODO: rename this to `kafka_producer' after alias + %% support is added to hocon; keeping this as just `kafka' + %% for backwards compatibility. + <<"kafka">> => #{ summary => <<"Kafka Producer Bridge">>, value => values(Method) } @@ -171,7 +176,7 @@ fields(producer_opts) -> %% Note: there's an implicit convention in `emqx_bridge' that, %% for egress bridges with this config, the published messages %% will be forwarded to such bridges. - {local_topic, mk(binary(), #{desc => ?DESC(mqtt_topic)})}, + {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})}, {kafka, mk(ref(producer_kafka_opts), #{ required => true, @@ -332,10 +337,33 @@ struct_names() -> %% internal type_field() -> {type, - mk(enum([kafka_consumer, kafka_producer]), #{required => true, desc => ?DESC("desc_type")})}. + %% TODO: rename `kafka' to `kafka_producer' after alias + %% support is added to hocon; keeping this as just `kafka' for + %% backwards compatibility. + mk(enum([kafka_consumer, kafka]), #{required => true, desc => ?DESC("desc_type")})}. name_field() -> {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. ref(Name) -> hoconsc:ref(?MODULE, Name). + +kafka_producer_converter(undefined, _HoconOpts) -> + undefined; +kafka_producer_converter( + #{<<"producer">> := OldOpts0, <<"bootstrap_hosts">> := _} = Config0, _HoconOpts +) -> + %% old schema + MQTTOpts = maps:get(<<"mqtt">>, OldOpts0, #{}), + LocalTopic = maps:get(<<"topic">>, MQTTOpts, undefined), + KafkaOpts = maps:get(<<"kafka">>, OldOpts0), + Config = maps:without([<<"producer">>], Config0), + case LocalTopic =:= undefined of + true -> + Config#{<<"kafka">> => KafkaOpts}; + false -> + Config#{<<"kafka">> => KafkaOpts, <<"local_topic">> => LocalTopic} + end; +kafka_producer_converter(Config, _HoconOpts) -> + %% new schema + Config. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index 785825a24..d46f687dd 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -22,6 +22,10 @@ -include_lib("emqx/include/logger.hrl"). +%% TODO: rename this to `kafka_producer' after alias support is added +%% to hocon; keeping this as just `kafka' for backwards compatibility. +-define(BRIDGE_TYPE, kafka). + callback_mode() -> async_if_possible. %% @doc Config schema is defined in emqx_ee_bridge_kafka. @@ -37,12 +41,11 @@ on_start(InstId, Config) -> socket_opts := SocketOpts, ssl := SSL } = Config, - BridgeType = kafka_consumer, + BridgeType = ?BRIDGE_TYPE, ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), _ = maybe_install_wolff_telemetry_handlers(ResourceId), Hosts = emqx_bridge_impl_kafka:hosts(Hosts0), - KafkaType = kafka_producer, - ClientId = emqx_bridge_impl_kafka:make_client_id(KafkaType, BridgeName), + ClientId = emqx_bridge_impl_kafka:make_client_id(BridgeType, BridgeName), ClientConfig = #{ min_metadata_refresh_interval => MinMetaRefreshInterval, connect_timeout => ConnTimeout, @@ -315,7 +318,7 @@ producers_config(BridgeName, ClientId, Input, IsDryRun) -> disk -> {false, replayq_dir(ClientId)}; hybrid -> {true, replayq_dir(ClientId)} end, - BridgeType = kafka_producer, + BridgeType = ?BRIDGE_TYPE, ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), #{ name => make_producer_name(BridgeName, IsDryRun), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 7a214de08..4b9642442 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -30,16 +30,15 @@ -include_lib("emqx/include/emqx.hrl"). -include("emqx_dashboard.hrl"). --define(CONTENT_TYPE, "application/x-www-form-urlencoded"). - -define(HOST, "http://127.0.0.1:18083"). %% -define(API_VERSION, "v5"). -define(BASE_PATH, "/api/v5"). --define(APP_DASHBOARD, emqx_dashboard). --define(APP_MANAGEMENT, emqx_management). +%% TODO: rename this to `kafka_producer' after alias support is added +%% to hocon; keeping this as just `kafka' for backwards compatibility. +-define(BRIDGE_TYPE, "kafka"). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -233,7 +232,7 @@ kafka_bridge_rest_api_all_auth_methods(UseSSL) -> ok. kafka_bridge_rest_api_helper(Config) -> - BridgeType = "kafka_producer", + BridgeType = ?BRIDGE_TYPE, BridgeName = "my_kafka_bridge", BridgeID = emqx_bridge_resource:bridge_id( erlang:list_to_binary(BridgeType), @@ -244,6 +243,7 @@ kafka_bridge_rest_api_helper(Config) -> erlang:list_to_binary(BridgeName) ), UrlEscColon = "%3A", + BridgesProbeParts = ["bridges_probe"], BridgeIdUrlEnc = BridgeType ++ UrlEscColon ++ BridgeName, BridgesParts = ["bridges"], BridgesPartsIdDeleteAlsoActions = ["bridges", BridgeIdUrlEnc ++ "?also_delete_dep_actions"], @@ -277,7 +277,7 @@ kafka_bridge_rest_api_helper(Config) -> %% Create new Kafka bridge KafkaTopic = "test-topic-one-partition", CreateBodyTmp = #{ - <<"type">> => <<"kafka_producer">>, + <<"type">> => <>, <<"name">> => <<"my_kafka_bridge">>, <<"bootstrap_hosts">> => iolist_to_binary(maps:get(<<"bootstrap_hosts">>, Config)), <<"enable">> => true, @@ -300,6 +300,13 @@ kafka_bridge_rest_api_helper(Config) -> {ok, 201, _Data} = show(http_post(BridgesParts, show(CreateBody))), %% Check that the new bridge is in the list of bridges true = MyKafkaBridgeExists(), + %% Probe should work + {ok, 204, _} = http_post(BridgesProbeParts, CreateBody), + %% no extra atoms should be created when probing + AtomsBefore = erlang:system_info(atom_count), + {ok, 204, _} = http_post(BridgesProbeParts, CreateBody), + AtomsAfter = erlang:system_info(atom_count), + ?assertEqual(AtomsBefore, AtomsAfter), %% Create a rule that uses the bridge {ok, 201, _Rule} = http_post( ["rules"], @@ -377,10 +384,10 @@ t_failed_creation_then_fix(Config) -> ValidAuthSettings = valid_sasl_plain_settings(), WrongAuthSettings = ValidAuthSettings#{"password" := "wrong"}, Hash = erlang:phash2([HostsString, ?FUNCTION_NAME]), - Type = kafka_producer, + Type = ?BRIDGE_TYPE, Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash), - ResourceId = emqx_bridge_resource:resource_id("kafka_producer", Name), - BridgeId = emqx_bridge_resource:bridge_id("kafka_producer", Name), + ResourceId = emqx_bridge_resource:resource_id(Type, Name), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), KafkaTopic = "test-topic-one-partition", WrongConf = config(#{ "authentication" => WrongAuthSettings, @@ -511,7 +518,7 @@ publish_helper( end, Hash = erlang:phash2([HostsString, AuthSettings, SSLSettings]), Name = "kafka_bridge_name_" ++ erlang:integer_to_list(Hash), - Type = "kafka_producer", + Type = ?BRIDGE_TYPE, InstId = emqx_bridge_resource:resource_id(Type, Name), KafkaTopic = "test-topic-one-partition", Conf = config( @@ -526,7 +533,7 @@ publish_helper( Conf0 ), {ok, _} = emqx_bridge:create( - <<"kafka_producer">>, list_to_binary(Name), Conf + <>, list_to_binary(Name), Conf ), Time = erlang:unique_integer(), BinTime = integer_to_binary(Time), @@ -600,8 +607,11 @@ hocon_config(Args) -> %% erlfmt-ignore hocon_config_template() -> +%% TODO: rename the type to `kafka_producer' after alias support is +%% added to hocon; keeping this as just `kafka' for backwards +%% compatibility. """ -bridges.kafka_producer.{{ bridge_name }} { +bridges.kafka.{{ bridge_name }} { bootstrap_hosts = \"{{ kafka_hosts_string }}\" enable = true authentication = {{{ authentication }}} diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl new file mode 100644 index 000000000..47c21b673 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl @@ -0,0 +1,181 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_kafka_tests). + +-include_lib("eunit/include/eunit.hrl"). + +%%=========================================================================== +%% Test cases +%%=========================================================================== + +kafka_producer_test() -> + Conf1 = parse(kafka_producer_old_hocon(_WithLocalTopic0 = false)), + Conf2 = parse(kafka_producer_old_hocon(_WithLocalTopic1 = true)), + Conf3 = parse(kafka_producer_new_hocon()), + + ?assertMatch( + #{ + <<"bridges">> := + #{ + <<"kafka">> := + #{ + <<"myproducer">> := + #{<<"kafka">> := #{}} + } + } + }, + check(Conf1) + ), + ?assertNotMatch( + #{ + <<"bridges">> := + #{ + <<"kafka">> := + #{ + <<"myproducer">> := + #{<<"local_topic">> := _} + } + } + }, + check(Conf1) + ), + ?assertMatch( + #{ + <<"bridges">> := + #{ + <<"kafka">> := + #{ + <<"myproducer">> := + #{ + <<"kafka">> := #{}, + <<"local_topic">> := <<"mqtt/local">> + } + } + } + }, + check(Conf2) + ), + ?assertMatch( + #{ + <<"bridges">> := + #{ + <<"kafka">> := + #{ + <<"myproducer">> := + #{ + <<"kafka">> := #{}, + <<"local_topic">> := <<"mqtt/local">> + } + } + } + }, + check(Conf3) + ), + + ok. + +%%=========================================================================== +%% Helper functions +%%=========================================================================== + +parse(Hocon) -> + {ok, Conf} = hocon:binary(Hocon), + Conf. + +check(Conf) when is_map(Conf) -> + hocon_tconf:check_plain(emqx_bridge_schema, Conf). + +%%=========================================================================== +%% Data section +%%=========================================================================== + +%% erlfmt-ignore +kafka_producer_old_hocon(_WithLocalTopic = true) -> + kafka_producer_old_hocon("mqtt {topic = \"mqtt/local\"}\n"); +kafka_producer_old_hocon(_WithLocalTopic = false) -> + kafka_producer_old_hocon("mqtt {}\n"); +kafka_producer_old_hocon(MQTTConfig) when is_list(MQTTConfig) -> +""" +bridges.kafka { + myproducer { + authentication = \"none\" + bootstrap_hosts = \"toxiproxy:9292\" + connect_timeout = \"5s\" + metadata_request_timeout = \"5s\" + min_metadata_refresh_interval = \"3s\" + producer { + kafka { + buffer { + memory_overload_protection = false + mode = \"memory\" + per_partition_limit = \"2GB\" + segment_bytes = \"100MB\" + } + compression = \"no_compression\" + max_batch_bytes = \"896KB\" + max_inflight = 10 + message { + key = \"${.clientid}\" + timestamp = \"${.timestamp}\" + value = \"${.}\" + } + partition_count_refresh_interval = \"60s\" + partition_strategy = \"random\" + required_acks = \"all_isr\" + topic = \"test-topic-two-partitions\" + } +""" ++ MQTTConfig ++ +""" + } + socket_opts { + nodelay = true + recbuf = \"1024KB\" + sndbuf = \"1024KB\" + } + ssl {enable = false, verify = \"verify_peer\"} + } +} +""". + +kafka_producer_new_hocon() -> + "" + "\n" + "bridges.kafka {\n" + " myproducer {\n" + " authentication = \"none\"\n" + " bootstrap_hosts = \"toxiproxy:9292\"\n" + " connect_timeout = \"5s\"\n" + " metadata_request_timeout = \"5s\"\n" + " min_metadata_refresh_interval = \"3s\"\n" + " kafka {\n" + " buffer {\n" + " memory_overload_protection = false\n" + " mode = \"memory\"\n" + " per_partition_limit = \"2GB\"\n" + " segment_bytes = \"100MB\"\n" + " }\n" + " compression = \"no_compression\"\n" + " max_batch_bytes = \"896KB\"\n" + " max_inflight = 10\n" + " message {\n" + " key = \"${.clientid}\"\n" + " timestamp = \"${.timestamp}\"\n" + " value = \"${.}\"\n" + " }\n" + " partition_count_refresh_interval = \"60s\"\n" + " partition_strategy = \"random\"\n" + " required_acks = \"all_isr\"\n" + " topic = \"test-topic-two-partitions\"\n" + " }\n" + " local_topic = \"mqtt/local\"\n" + " socket_opts {\n" + " nodelay = true\n" + " recbuf = \"1024KB\"\n" + " sndbuf = \"1024KB\"\n" + " }\n" + " ssl {enable = false, verify = \"verify_peer\"}\n" + " }\n" + "}\n" + "". From 53979b626104e076e28990f25888586ee4d063fa Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 10 Mar 2023 10:28:06 -0300 Subject: [PATCH 74/88] feat(kafka_consumer): support multiple topic mappings, payload templates and key/value encodings Added after feedback from the product team. --- .../i18n/emqx_ee_bridge_kafka.conf | 59 +++- .../src/emqx_ee_bridge_kafka.erl | 50 ++- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 193 ++++++---- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 333 +++++++++++++++--- 4 files changed, 509 insertions(+), 126 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 636573d07..ed88a1e0d 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -536,17 +536,33 @@ emqx_ee_bridge_kafka { } consumer_mqtt_payload { desc { - en: "The payload of the MQTT message to be published.\n" - "full_message will encode available Kafka message attributes as a JSON object, including Key, Value, Timestamp and Headers" - "message_value will directly use the Kafka message value as the " - "MQTT message payload." - zh: "要发布的MQTT消息的有效载荷。" - "full_message将把所有可用数据编码为JSON对象，包括 Key，Value，Timestamp 和 Headers。" - "message_value将直接使用 Kafka 消息值作为MQTT消息的 Payload。" + en: "The template for transforming the incoming Kafka message." + " By default, it will use JSON format to serialize all visible" + " inputs from the Kafka message. Such fields are:\n" + "headers: an object containing string key-value pairs.\n" + "key: Kafka message key (uses the chosen key encoding).\n" + "offset: offset for the message.\n" + "topic: Kafka topic.\n" + "ts: message timestamp.\n" + "ts_type: message timestamp type, which is one of" + " create, append or undefined.\n" + "value: Kafka message value (uses the chosen value encoding).\n" + zh: "用于转换传入的Kafka消息的模板。 " + "默认情况下，它将使用JSON格式来序列化所有来自Kafka消息的可见输入。 " + "这样的字段是。" + "headers: 一个包含字符串键值对的对象。\n" + "key: Kafka消息密钥（使用选择的密钥编码）。\n" + "offset: 信息的偏移量。\n" + "topic: 卡夫卡主题。\n" + "ts: 消息的时间戳。\n" + "ts_type: 消息的时间戳类型，它是一个" + " create， append 或 undefined。\n" + "value: Kafka消息值（使用选择的值编码）。\n" + } label { - en: "MQTT Payload" - zh: "MQTT Payload" + en: "MQTT Payload Template" + zh: "MQTT Payload Template" } } consumer_kafka_topic { @@ -603,4 +619,29 @@ emqx_ee_bridge_kafka { zh: "偏移承诺间隔" } } + consumer_topic_mapping { + desc { + en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." + zh: "定义了Kafka主题和MQTT主题之间的映射。 必须至少包含一个项目。" + } + label { + en: "Topic Mapping" + zh: "主题图" + } + } + consumer_encoding_mode { + desc { + en: "Defines how the key or value from the Kafka message is" + " dealt with before being forwarded via MQTT.\n" + "force_utf8 Uses UTF-8 encoding directly from the original message.\n" + "base64 Uses base-64 encoding on the received key or value." + zh: "定义了在通过MQTT转发之前如何处理Kafka消息的键或值。" + "force_utf8 直接使用原始信息的UTF-8编码。\n" + "base64 对收到的密钥或值使用base-64编码。" + } + label { + en: "Encoding Mode" + zh: "编码模式" + } + } } diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 583acc48d..de67a73c6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -263,26 +263,44 @@ fields(producer_buffer) -> fields(consumer_opts) -> [ {kafka, - mk(ref(consumer_kafka_opts), #{required => true, desc => ?DESC(consumer_kafka_opts)})}, - {mqtt, mk(ref(consumer_mqtt_opts), #{required => true, desc => ?DESC(consumer_mqtt_opts)})} - ]; -fields(consumer_mqtt_opts) -> - [ - {topic, - mk(binary(), #{ - required => true, - desc => ?DESC(consumer_mqtt_topic) - })}, - {qos, mk(emqx_schema:qos(), #{default => 0, desc => ?DESC(consumer_mqtt_qos)})}, - {payload, + mk(ref(consumer_kafka_opts), #{required => false, desc => ?DESC(consumer_kafka_opts)})}, + {topic_mapping, mk( - enum([full_message, message_value]), - #{default => full_message, desc => ?DESC(consumer_mqtt_payload)} + hoconsc:array(ref(consumer_topic_mapping)), + #{ + required => true, + desc => ?DESC(consumer_topic_mapping), + validator => + fun + ([]) -> + {error, "There must be at least one Kafka-MQTT topic mapping"}; + ([_ | _]) -> + ok + end + } + )}, + {key_encoding_mode, + mk(enum([force_utf8, base64]), #{ + default => force_utf8, desc => ?DESC(consumer_encoding_mode) + })}, + {value_encoding_mode, + mk(enum([force_utf8, base64]), #{ + default => force_utf8, desc => ?DESC(consumer_encoding_mode) + })} + ]; +fields(consumer_topic_mapping) -> + [ + {kafka_topic, mk(binary(), #{required => true, desc => ?DESC(consumer_kafka_topic)})}, + {mqtt_topic, mk(binary(), #{required => true, desc => ?DESC(consumer_mqtt_topic)})}, + {qos, mk(emqx_schema:qos(), #{default => 0, desc => ?DESC(consumer_mqtt_qos)})}, + {payload_template, + mk( + string(), + #{default => <<"${.}">>, desc => ?DESC(consumer_mqtt_payload)} )} ]; fields(consumer_kafka_opts) -> [ - {topic, mk(binary(), #{desc => ?DESC(consumer_kafka_topic)})}, {max_batch_bytes, mk(emqx_schema:bytesize(), #{ default => "896KB", desc => ?DESC(consumer_max_batch_bytes) @@ -330,7 +348,7 @@ struct_names() -> producer_opts, consumer_opts, consumer_kafka_opts, - consumer_mqtt_opts + consumer_topic_mapping ]. %% ------------------------------------------------------------------------------------------------- diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index 43717dd89..99877ca8e 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -41,31 +41,59 @@ offset_reset_policy := offset_reset_policy(), topic := binary() }, - mqtt := #{ - topic := emqx_types:topic(), - qos := emqx_types:qos(), - payload := mqtt_payload() - }, + topic_mapping := nonempty_list( + #{ + kafka_topic := kafka_topic(), + mqtt_topic := emqx_types:topic(), + qos := emqx_types:qos(), + payload_template := string() + } + ), ssl := _, any() => term() }. -type subscriber_id() :: emqx_ee_bridge_kafka_consumer_sup:child_id(). +-type kafka_topic() :: brod:topic(). -type state() :: #{ - kafka_topic := binary(), + kafka_topics := nonempty_list(kafka_topic()), subscriber_id := subscriber_id(), kafka_client_id := brod:client_id() }. -type offset_reset_policy() :: reset_to_latest | reset_to_earliest | reset_by_subscriber. --type mqtt_payload() :: full_message | message_value. --type consumer_state() :: #{ - resource_id := resource_id(), - mqtt := #{ - payload := mqtt_payload(), - topic => emqx_types:topic(), - qos => emqx_types:qos() - }, +%% -type mqtt_payload() :: full_message | message_value. +-type encoding_mode() :: force_utf8 | base64. +-type consumer_init_data() :: #{ hookpoint := binary(), - kafka_topic := binary() + key_encoding_mode := encoding_mode(), + resource_id := resource_id(), + topic_mapping := #{ + kafka_topic() := #{ + payload_template := emqx_plugin_libs_rule:tmpl_token(), + mqtt_topic => emqx_types:topic(), + qos => emqx_types:qos() + } + }, + value_encoding_mode := encoding_mode() +}. +-type consumer_state() :: #{ + hookpoint := binary(), + kafka_topic := binary(), + key_encoding_mode := encoding_mode(), + resource_id := resource_id(), + topic_mapping := #{ + kafka_topic() := #{ + payload_template := emqx_plugin_libs_rule:tmpl_token(), + mqtt_topic => emqx_types:topic(), + qos => emqx_types:qos() + } + }, + value_encoding_mode := encoding_mode() +}. +-type subscriber_init_info() :: #{ + topic => brod:topic(), + parition => brod:partition(), + group_id => brod:group_id(), + commit_fun => brod_group_subscriber_v2:commit_fun() }. %%------------------------------------------------------------------------------------- @@ -92,11 +120,10 @@ on_start(InstanceId, Config) -> max_batch_bytes := _, max_rejoin_attempts := _, offset_commit_interval_seconds := _, - offset_reset_policy := _, - topic := _ + offset_reset_policy := _ }, - mqtt := #{topic := _, qos := _, payload := _}, - ssl := SSL + ssl := SSL, + topic_mapping := _ } = Config, BootstrapHosts = emqx_bridge_impl_kafka:hosts(BootstrapHosts0), KafkaType = kafka_consumer, @@ -145,22 +172,19 @@ on_get_status(_InstanceID, State) -> #{ subscriber_id := SubscriberId, kafka_client_id := ClientID, - kafka_topic := KafkaTopic + kafka_topics := KafkaTopics } = State, - case brod:get_partitions_count(ClientID, KafkaTopic) of - {ok, NPartitions} -> - do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions); - _ -> - disconnected - end. + do_get_status(ClientID, KafkaTopics, SubscriberId). %%------------------------------------------------------------------------------------- %% `brod_group_subscriber' API %%------------------------------------------------------------------------------------- --spec init(_, consumer_state()) -> {ok, consumer_state()}. -init(_GroupData, State) -> - ?tp(kafka_consumer_subscriber_init, #{group_data => _GroupData, state => State}), +-spec init(subscriber_init_info(), consumer_init_data()) -> {ok, consumer_state()}. +init(GroupData, State0) -> + ?tp(kafka_consumer_subscriber_init, #{group_data => GroupData, state => State0}), + #{topic := KafkaTopic} = GroupData, + State = State0#{kafka_topic => KafkaTopic}, {ok, State}. -spec handle_message(#kafka_message{}, consumer_state()) -> {ok, commit, consumer_state()}. @@ -173,33 +197,29 @@ handle_message(Message, State) -> do_handle_message(Message, State) -> #{ - resource_id := ResourceId, hookpoint := Hookpoint, kafka_topic := KafkaTopic, - mqtt := #{ - topic := MQTTTopic, - payload := MQTTPayload, - qos := MQTTQoS - } + key_encoding_mode := KeyEncodingMode, + resource_id := ResourceId, + topic_mapping := TopicMapping, + value_encoding_mode := ValueEncodingMode } = State, + #{ + mqtt_topic := MQTTTopic, + qos := MQTTQoS, + payload_template := PayloadTemplate + } = maps:get(KafkaTopic, TopicMapping), FullMessage = #{ + headers => maps:from_list(Message#kafka_message.headers), + key => encode(Message#kafka_message.key, KeyEncodingMode), offset => Message#kafka_message.offset, - key => Message#kafka_message.key, - value => Message#kafka_message.value, + topic => KafkaTopic, ts => Message#kafka_message.ts, ts_type => Message#kafka_message.ts_type, - headers => maps:from_list(Message#kafka_message.headers), - topic => KafkaTopic + value => encode(Message#kafka_message.value, ValueEncodingMode) }, - Payload = - case MQTTPayload of - full_message -> - FullMessage; - message_value -> - Message#kafka_message.value - end, - EncodedPayload = emqx_json:encode(Payload), - MQTTMessage = emqx_message:make(ResourceId, MQTTQoS, MQTTTopic, EncodedPayload), + Payload = render(FullMessage, PayloadTemplate), + MQTTMessage = emqx_message:make(ResourceId, MQTTQoS, MQTTTopic, Payload), _ = emqx:publish(MQTTMessage), emqx:run_hook(Hookpoint, [FullMessage]), emqx_resource_metrics:received_inc(ResourceId), @@ -251,21 +271,20 @@ start_consumer(Config, InstanceId, ClientID) -> max_batch_bytes := MaxBatchBytes, max_rejoin_attempts := MaxRejoinAttempts, offset_commit_interval_seconds := OffsetCommitInterval, - offset_reset_policy := OffsetResetPolicy, - topic := KafkaTopic + offset_reset_policy := OffsetResetPolicy }, - mqtt := #{topic := MQTTTopic, qos := MQTTQoS, payload := MQTTPayload} + key_encoding_mode := KeyEncodingMode, + topic_mapping := TopicMapping0, + value_encoding_mode := ValueEncodingMode } = Config, ok = ensure_consumer_supervisor_started(), + TopicMapping = convert_topic_mapping(TopicMapping0), InitialState = #{ - resource_id => emqx_bridge_resource:resource_id(kafka_consumer, BridgeName), - mqtt => #{ - payload => MQTTPayload, - topic => MQTTTopic, - qos => MQTTQoS - }, + key_encoding_mode => KeyEncodingMode, hookpoint => Hookpoint, - kafka_topic => KafkaTopic + resource_id => emqx_bridge_resource:resource_id(kafka_consumer, BridgeName), + topic_mapping => TopicMapping, + value_encoding_mode => ValueEncodingMode }, %% note: the group id should be the same for all nodes in the %% cluster, so that the load gets distributed between all @@ -279,11 +298,12 @@ start_consumer(Config, InstanceId, ClientID) -> {max_rejoin_attempts, MaxRejoinAttempts}, {offset_commit_interval_seconds, OffsetCommitInterval} ], + KafkaTopics = maps:keys(TopicMapping), GroupSubscriberConfig = #{ client => ClientID, group_id => GroupID, - topics => [KafkaTopic], + topics => KafkaTopics, cb_module => ?MODULE, init_data => InitialState, message_type => message, @@ -304,14 +324,13 @@ start_consumer(Config, InstanceId, ClientID) -> {ok, #{ subscriber_id => SubscriberId, kafka_client_id => ClientID, - kafka_topic => KafkaTopic + kafka_topics => KafkaTopics }}; {error, Reason2} -> ?SLOG(error, #{ msg => "failed_to_start_kafka_consumer", instance_id => InstanceId, kafka_hosts => emqx_bridge_impl_kafka:hosts(BootstrapHosts0), - kafka_topic => KafkaTopic, reason => emqx_misc:redact(Reason2) }), stop_client(ClientID), @@ -344,6 +363,19 @@ stop_client(ClientID) -> ), ok. +do_get_status(ClientID, [KafkaTopic | RestTopics], SubscriberId) -> + case brod:get_partitions_count(ClientID, KafkaTopic) of + {ok, NPartitions} -> + case do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions) of + connected -> do_get_status(ClientID, RestTopics, SubscriberId); + disconnected -> disconnected + end; + _ -> + disconnected + end; +do_get_status(_ClientID, _KafkaTopics = [], _SubscriberId) -> + connected. + -spec do_get_status(brod:client_id(), binary(), subscriber_id(), pos_integer()) -> connected | disconnected. do_get_status(ClientID, KafkaTopic, SubscriberId, NPartitions) -> @@ -424,5 +456,44 @@ make_client_id(InstanceId, KafkaType, KafkaName) -> probing_brod_consumers end. +convert_topic_mapping(TopicMappingList) -> + lists:foldl( + fun(Fields, Acc) -> + #{ + kafka_topic := KafkaTopic, + mqtt_topic := MQTTTopic, + qos := QoS, + payload_template := PayloadTemplate0 + } = Fields, + PayloadTemplate = emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate0), + Acc#{ + KafkaTopic => #{ + payload_template => PayloadTemplate, + mqtt_topic => MQTTTopic, + qos => QoS + } + } + end, + #{}, + TopicMappingList + ). + +render(FullMessage, PayloadTemplate) -> + Opts = #{ + return => full_binary, + var_trans => fun + (undefined) -> + <<>>; + (X) -> + emqx_plugin_libs_rule:bin(X) + end + }, + emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, FullMessage, Opts). + +encode(Value, force_utf8) -> + Value; +encode(Value, base64) -> + base64:encode(Value). + to_bin(B) when is_binary(B) -> B; to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index e0be06e1a..129011862 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -10,6 +10,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -53,7 +54,8 @@ sasl_only_tests() -> only_once_tests() -> [ t_bridge_rule_action_source, - t_cluster_group + t_cluster_group, + t_multiple_topic_mappings ]. init_per_suite(Config) -> @@ -284,6 +286,32 @@ init_per_testcase(TestCase, Config) when init_per_testcase(t_cluster_group = TestCase, Config0) -> Config = emqx_misc:merge_opts(Config0, [{num_partitions, 6}]), common_init_per_testcase(TestCase, Config); +init_per_testcase(t_multiple_topic_mappings = TestCase, Config0) -> + KafkaTopicBase = + << + (atom_to_binary(TestCase))/binary, + (integer_to_binary(erlang:unique_integer()))/binary + >>, + MQTTTopicBase = + <<"mqtt/", (atom_to_binary(TestCase))/binary, + (integer_to_binary(erlang:unique_integer()))/binary, "/">>, + TopicMapping = + [ + #{ + kafka_topic => <>, + mqtt_topic => <>, + qos => 1, + payload_template => <<"${.}">> + }, + #{ + kafka_topic => <>, + mqtt_topic => <>, + qos => 2, + payload_template => <<"v = ${.value}">> + } + ], + Config = [{topic_mapping, TopicMapping} | Config0], + common_init_per_testcase(TestCase, Config); init_per_testcase(TestCase, Config) -> common_init_per_testcase(TestCase, Config). @@ -295,23 +323,35 @@ common_init_per_testcase(TestCase, Config0) -> (atom_to_binary(TestCase))/binary, (integer_to_binary(erlang:unique_integer()))/binary >>, - Config = [{kafka_topic, KafkaTopic} | Config0], - KafkaType = ?config(kafka_type, Config), + KafkaType = ?config(kafka_type, Config0), + UniqueNum = integer_to_binary(erlang:unique_integer()), + MQTTTopic = proplists:get_value(mqtt_topic, Config0, <<"mqtt/topic/", UniqueNum/binary>>), + MQTTQoS = proplists:get_value(mqtt_qos, Config0, 0), + DefaultTopicMapping = [ + #{ + kafka_topic => KafkaTopic, + mqtt_topic => MQTTTopic, + qos => MQTTQoS, + payload_template => <<"${.}">> + } + ], + TopicMapping = proplists:get_value(topic_mapping, Config0, DefaultTopicMapping), + Config = [ + {kafka_topic, KafkaTopic}, + {topic_mapping, TopicMapping} + | Config0 + ], {Name, ConfigString, KafkaConfig} = kafka_config( TestCase, KafkaType, Config ), - ensure_topic(Config), - #{ - producers := Producers, - clientid := KafkaProducerClientId - } = start_producer(TestCase, Config), + ensure_topics(Config), + ProducersConfigs = start_producers(TestCase, Config), ok = snabbkaffe:start_trace(), [ {kafka_name, Name}, {kafka_config_string, ConfigString}, {kafka_config, KafkaConfig}, - {kafka_producers, Producers}, - {kafka_producer_clientid, KafkaProducerClientId} + {kafka_producers, ProducersConfigs} | Config ]. @@ -323,11 +363,17 @@ end_per_testcase(_Testcase, Config) -> false -> ProxyHost = ?config(proxy_host, Config), ProxyPort = ?config(proxy_port, Config), - Producers = ?config(kafka_producers, Config), - KafkaProducerClientId = ?config(kafka_producer_clientid, Config), + ProducersConfigs = ?config(kafka_producers, Config), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), delete_all_bridges(), - ok = wolff:stop_and_delete_supervised_producers(Producers), + #{clientid := KafkaProducerClientId, producers := ProducersMapping} = + ProducersConfigs, + lists:foreach( + fun(Producers) -> + ok = wolff:stop_and_delete_supervised_producers(Producers) + end, + maps:values(ProducersMapping) + ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), emqx_common_test_helpers:call_janitor(), ok = snabbkaffe:stop(), @@ -338,8 +384,8 @@ end_per_testcase(_Testcase, Config) -> %% Helper fns %%------------------------------------------------------------------------------ -start_producer(TestCase, Config) -> - KafkaTopic = ?config(kafka_topic, Config), +start_producers(TestCase, Config) -> + TopicMapping = ?config(topic_mapping, Config), KafkaClientId = <<"test-client-", (atom_to_binary(TestCase))/binary, (integer_to_binary(erlang:unique_integer()))/binary>>, @@ -381,9 +427,27 @@ start_producer(TestCase, Config) -> ssl => SSL }, {ok, Clients} = wolff:ensure_supervised_client(KafkaClientId, Hosts, ClientConfig), + ProducersData0 = + #{ + clients => Clients, + clientid => KafkaClientId, + producers => #{} + }, + lists:foldl( + fun(#{kafka_topic := KafkaTopic}, #{producers := ProducersMapping0} = Acc) -> + Producers = do_start_producer(KafkaClientId, KafkaTopic), + ProducersMapping = ProducersMapping0#{KafkaTopic => Producers}, + Acc#{producers := ProducersMapping} + end, + ProducersData0, + TopicMapping + ). + +do_start_producer(KafkaClientId, KafkaTopic) -> + Name = binary_to_atom(<>), ProducerConfig = #{ - name => test_producer, + name => Name, partitioner => roundrobin, partition_count_refresh_interval_seconds => 1_000, replayq_max_total_bytes => 10_000, @@ -396,14 +460,10 @@ start_producer(TestCase, Config) -> telemetry_meta_data => #{} }, {ok, Producers} = wolff:ensure_supervised_producers(KafkaClientId, KafkaTopic, ProducerConfig), - #{ - producers => Producers, - clients => Clients, - clientid => KafkaClientId - }. + Producers. -ensure_topic(Config) -> - KafkaTopic = ?config(kafka_topic, Config), +ensure_topics(Config) -> + TopicMapping = ?config(topic_mapping, Config), KafkaHost = ?config(kafka_host, Config), KafkaPort = ?config(kafka_port, Config), UseTLS = ?config(use_tls, Config), @@ -418,6 +478,7 @@ ensure_topic(Config) -> assignments => [], configs => [] } + || #{kafka_topic := KafkaTopic} <- TopicMapping ], RequestConfig = #{timeout => 5_000}, ConnConfig0 = @@ -464,8 +525,16 @@ shared_secret(rig_keytab) -> filename:join([shared_secret_path(), "rig.keytab"]). publish(Config, Messages) -> - Producers = ?config(kafka_producers, Config), - ct:pal("publishing: ~p", [Messages]), + %% pick the first topic if not specified + #{producers := ProducersMapping} = ?config(kafka_producers, Config), + [{KafkaTopic, Producers} | _] = maps:to_list(ProducersMapping), + ct:pal("publishing to ~p:\n ~p", [KafkaTopic, Messages]), + {_Partition, _OffsetReply} = wolff:send_sync(Producers, Messages, 10_000). + +publish(Config, KafkaTopic, Messages) -> + #{producers := ProducersMapping} = ?config(kafka_producers, Config), + #{KafkaTopic := Producers} = ProducersMapping, + ct:pal("publishing to ~p:\n ~p", [KafkaTopic, Messages]), {_Partition, _OffsetReply} = wolff:send_sync(Producers, Messages, 10_000). kafka_config(TestCase, _KafkaType, Config) -> @@ -480,7 +549,16 @@ kafka_config(TestCase, _KafkaType, Config) -> >>, MQTTTopic = proplists:get_value(mqtt_topic, Config, <<"mqtt/topic/", UniqueNum/binary>>), MQTTQoS = proplists:get_value(mqtt_qos, Config, 0), - MQTTPayload = proplists:get_value(mqtt_payload, Config, full_message), + DefaultTopicMapping = [ + #{ + kafka_topic => KafkaTopic, + mqtt_topic => MQTTTopic, + qos => MQTTQoS, + payload_template => <<"${.}">> + } + ], + TopicMapping0 = proplists:get_value(topic_mapping, Config, DefaultTopicMapping), + TopicMappingStr = topic_mapping(TopicMapping0), ConfigString = io_lib:format( "bridges.kafka_consumer.~s {\n" @@ -491,18 +569,15 @@ kafka_config(TestCase, _KafkaType, Config) -> " metadata_request_timeout = 5s\n" "~s" " kafka {\n" - " topic = ~s\n" " max_batch_bytes = 896KB\n" " max_rejoin_attempts = 5\n" " offset_commit_interval_seconds = 3\n" %% todo: matrix this " offset_reset_policy = reset_to_latest\n" " }\n" - " mqtt {\n" - " topic = \"~s\"\n" - " qos = ~b\n" - " payload = ~p\n" - " }\n" + "~s" + " key_encoding_mode = force_utf8\n" + " value_encoding_mode = force_utf8\n" " ssl {\n" " enable = ~p\n" " verify = verify_none\n" @@ -514,15 +589,35 @@ kafka_config(TestCase, _KafkaType, Config) -> KafkaHost, KafkaPort, authentication(AuthType), - KafkaTopic, - MQTTTopic, - MQTTQoS, - MQTTPayload, + TopicMappingStr, UseTLS ] ), {Name, ConfigString, parse_and_check(ConfigString, Name)}. +topic_mapping(TopicMapping0) -> + Template0 = << + "{kafka_topic = \"{{ kafka_topic }}\"," + " mqtt_topic = \"{{ mqtt_topic }}\"," + " qos = {{ qos }}," + " payload_template = \"{{{ payload_template }}}\" }" + >>, + Template = bbmustache:parse_binary(Template0), + Entries = + lists:map( + fun(Params) -> + bbmustache:compile(Template, Params, [{key_type, atom}]) + end, + TopicMapping0 + ), + iolist_to_binary( + [ + " topic_mapping = [", + lists:join(<<",\n">>, Entries), + "]\n" + ] + ). + authentication(Type) when Type =:= scram_sha_256; Type =:= scram_sha_512; @@ -578,6 +673,29 @@ delete_all_bridges() -> emqx_bridge:list() ). +create_bridge_api(Config) -> + create_bridge_api(Config, _Overrides = #{}). + +create_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(kafka_name, Config), + KafkaConfig0 = ?config(kafka_config, Config), + KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("creating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {Status, Headers, Body0}} -> + {ok, {Status, Headers, emqx_json:decode(Body0, [return_maps])}}; + Error -> + Error + end, + ct:pal("bridge create result: ~p", [Res]), + Res. + update_bridge_api(Config) -> update_bridge_api(Config, _Overrides = #{}). @@ -702,15 +820,24 @@ wait_until_subscribers_are_ready(N, Timeout) -> %% flaky about when they decide truly consuming the messages... %% `Period' should be greater than the `sleep_timeout' of the consumer %% (default 1 s). -ping_until_healthy(_Config, _Period, Timeout) when Timeout =< 0 -> - ct:fail("kafka subscriber did not stabilize!"); ping_until_healthy(Config, Period, Timeout) -> + #{producers := ProducersMapping} = ?config(kafka_producers, Config), + [KafkaTopic | _] = maps:keys(ProducersMapping), + ping_until_healthy(Config, KafkaTopic, Period, Timeout). + +ping_until_healthy(_Config, _KafkaTopic, _Period, Timeout) when Timeout =< 0 -> + ct:fail("kafka subscriber did not stabilize!"); +ping_until_healthy(Config, KafkaTopic, Period, Timeout) -> TimeA = erlang:monotonic_time(millisecond), Payload = emqx_guid:to_hexstr(emqx_guid:gen()), - publish(Config, [#{key => <<"probing">>, value => Payload}]), + publish(Config, KafkaTopic, [#{key => <<"probing">>, value => Payload}]), Res = ?block_until( - #{?snk_kind := kafka_consumer_handle_message, ?snk_span := {complete, _}}, + #{ + ?snk_kind := kafka_consumer_handle_message, + ?snk_span := {complete, _}, + message := #kafka_message{value = Payload} + }, Period ), case Res of @@ -930,6 +1057,132 @@ t_start_and_consume_ok(Config) -> ), ok. +t_multiple_topic_mappings(Config) -> + TopicMapping = ?config(topic_mapping, Config), + MQTTTopics = [MQTTTopic || #{mqtt_topic := MQTTTopic} <- TopicMapping], + KafkaTopics = [KafkaTopic || #{kafka_topic := KafkaTopic} <- TopicMapping], + NumMQTTTopics = length(MQTTTopics), + NPartitions = ?config(num_partitions, Config), + ResourceId = resource_id(Config), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + ?check_trace( + begin + ?assertMatch( + {ok, {{_, 201, _}, _, _}}, + create_bridge_api(Config) + ), + wait_until_subscribers_are_ready(NPartitions, 40_000), + lists:foreach( + fun(KafkaTopic) -> + ping_until_healthy(Config, KafkaTopic, _Period = 1_500, _Timeout = 24_000) + end, + KafkaTopics + ), + + {ok, C} = emqtt:start_link([{proto_ver, v5}]), + on_exit(fun() -> emqtt:stop(C) end), + {ok, _} = emqtt:connect(C), + lists:foreach( + fun(MQTTTopic) -> + %% we use the hightest QoS so that we can check what + %% the subscription was. + QoS2Granted = 2, + {ok, _, [QoS2Granted]} = emqtt:subscribe(C, MQTTTopic, ?QOS_2) + end, + MQTTTopics + ), + + {ok, SRef0} = + snabbkaffe:subscribe( + ?match_event(#{ + ?snk_kind := kafka_consumer_handle_message, ?snk_span := {complete, _} + }), + NumMQTTTopics, + _Timeout0 = 20_000 + ), + lists:foreach( + fun(KafkaTopic) -> + publish(Config, KafkaTopic, [ + #{ + key => <<"mykey">>, + value => Payload, + headers => [{<<"hkey">>, <<"hvalue">>}] + } + ]) + end, + KafkaTopics + ), + {ok, _} = snabbkaffe:receive_events(SRef0), + + %% Check that the bridge probe API doesn't leak atoms. + ProbeRes = probe_bridge_api(Config), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + AtomsBefore = erlang:system_info(atom_count), + %% Probe again; shouldn't have created more atoms. + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + AtomsAfter = erlang:system_info(atom_count), + ?assertEqual(AtomsBefore, AtomsAfter), + + ok + end, + fun(Trace) -> + %% two messages processed with begin/end events + ?assertMatch([_, _, _, _ | _], ?of_kind(kafka_consumer_handle_message, Trace)), + Published = receive_published(#{n => NumMQTTTopics}), + lists:foreach( + fun( + #{ + mqtt_topic := MQTTTopic, + qos := MQTTQoS + } + ) -> + [Msg] = [ + Msg + || Msg = #{topic := T} <- Published, + T =:= MQTTTopic + ], + ?assertMatch( + #{ + qos := MQTTQoS, + topic := MQTTTopic, + payload := _ + }, + Msg + ) + end, + TopicMapping + ), + %% check that we observed the different payload templates + %% as configured. + Payloads = + lists:sort([ + case emqx_json:safe_decode(P, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> P + end + || #{payload := P} <- Published + ]), + ?assertMatch( + [ + #{ + <<"headers">> := #{<<"hkey">> := <<"hvalue">>}, + <<"key">> := <<"mykey">>, + <<"offset">> := Offset, + <<"topic">> := KafkaTopic, + <<"ts">> := TS, + <<"ts_type">> := <<"create">>, + <<"value">> := Payload + }, + <<"v = ", Payload/binary>> + ] when is_integer(Offset) andalso is_integer(TS) andalso is_binary(KafkaTopic), + Payloads + ), + ?assertEqual(2, emqx_resource_metrics:received_get(ResourceId)), + ok + end + ), + ok. + t_on_get_status(Config) -> case proplists:get_bool(skip_does_not_apply, Config) of true -> From 66eb4ef0691d26df4fdcedba50350a7318208ed1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 10 Mar 2023 17:08:57 -0300 Subject: [PATCH 75/88] test: fix inter-suite flakiness --- apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index 61df9bd29..f249aa95e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -38,7 +38,6 @@ init_per_suite(_Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - snabbkaffe:fix_ct_logging(), []. end_per_suite(_Config) -> From 37a35b22df60142726c1a6f08add3e5e01b46c92 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 14:02:43 -0300 Subject: [PATCH 76/88] docs(kafka): improve kafka consumer/producer API examples --- .../src/emqx_ee_bridge_kafka.erl | 80 +++++++++++++++++-- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index de67a73c6..943009945 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -43,25 +43,89 @@ conn_bridge_examples(Method) -> %% for backwards compatibility. <<"kafka">> => #{ summary => <<"Kafka Producer Bridge">>, - value => values(Method) + value => values({Method, producer}) } }, #{ <<"kafka_consumer">> => #{ summary => <<"Kafka Consumer Bridge">>, - value => values(Method) + value => values({Method, consumer}) } } ]. -values(get) -> - maps:merge(values(post), ?METRICS_EXAMPLE); -values(post) -> +values({get, KafkaType}) -> + maps:merge(values({post, KafkaType}), ?METRICS_EXAMPLE); +values({post, KafkaType}) -> + maps:merge(values(common_config), values(KafkaType)); +values({put, KafkaType}) -> + values({post, KafkaType}); +values(common_config) -> #{ - bootstrap_hosts => <<"localhost:9092">> + authentication => #{ + mechanism => <<"plain">>, + username => <<"username">>, + password => <<"password">> + }, + bootstrap_hosts => <<"localhost:9092">>, + connect_timeout => <<"5s">>, + enable => true, + metadata_request_timeout => <<"4s">>, + min_metadata_refresh_interval => <<"3s">>, + socket_opts => #{ + sndbuf => <<"1024KB">>, + recbuf => <<"1024KB">>, + nodelay => true + } }; -values(put) -> - values(post). +values(producer) -> + #{ + kafka => #{ + topic => <<"kafka-topic">>, + message => #{ + key => <<"${.clientid}">>, + value => <<"${.}">>, + timestamp => <<"${.timestamp}">> + }, + max_batch_bytes => <<"896KB">>, + compression => <<"no_compression">>, + partition_strategy => <<"random">>, + required_acks => <<"all_isr">>, + partition_count_refresh_interval => <<"60s">>, + max_inflight => 10, + buffer => #{ + mode => <<"hybrid">>, + per_partition_limit => <<"2GB">>, + segment_bytes => <<"100MB">>, + memory_overload_protection => true + } + }, + local_topic => <<"mqtt/local/topic">> + }; +values(consumer) -> + #{ + kafka => #{ + max_batch_bytes => <<"896KB">>, + offset_reset_policy => <<"reset_to_latest">>, + offset_commit_interval_seconds => 5 + }, + key_encoding_mode => <<"force_utf8">>, + topic_mapping => [ + #{ + kafka_topic => <<"kafka-topic-1">>, + mqtt_topic => <<"mqtt/topic/1">>, + qos => 1, + payload_template => <<"${.}">> + }, + #{ + kafka_topic => <<"kafka-topic-2">>, + mqtt_topic => <<"mqtt/topic/2">>, + qos => 2, + payload_template => <<"v = ${.value}">> + } + ], + value_encoding_mode => <<"force_utf8">> + }. %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions From fc5dfa108a048906038f0f8f79d222247a3629f3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 17:08:07 -0300 Subject: [PATCH 77/88] fix(kafka_consumer): rename `force_utf8` to `none` We actually enforce the key/value to be a valid UTF-8 string when using `emqx_json:encode`, if we do encode using that, which is template-dependent. --- .../i18n/emqx_ee_bridge_kafka.conf | 6 +- .../src/emqx_ee_bridge_kafka.erl | 33 +++--- .../kafka/emqx_bridge_impl_kafka_consumer.erl | 4 +- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 4 +- .../test/emqx_ee_bridge_kafka_tests.erl | 106 ++++++++++++++++++ 5 files changed, 134 insertions(+), 19 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index ed88a1e0d..2e4a58c7f 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -633,10 +633,12 @@ emqx_ee_bridge_kafka { desc { en: "Defines how the key or value from the Kafka message is" " dealt with before being forwarded via MQTT.\n" - "force_utf8 Uses UTF-8 encoding directly from the original message.\n" + "none Uses the key or value from the Kafka message unchanged." + " Note: in this case, then the key or value must be a valid UTF-8 string.\n" "base64 Uses base-64 encoding on the received key or value." zh: "定义了在通过MQTT转发之前如何处理Kafka消息的键或值。" - "force_utf8 直接使用原始信息的UTF-8编码。\n" + "none 使用Kafka消息中的键或值，不改变。" + " 注意：在这种情况下，那么键或值必须是一个有效的UTF-8字符串。\n" "base64 对收到的密钥或值使用base-64编码。" } label { diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 943009945..8e9ff9628 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -109,7 +109,7 @@ values(consumer) -> offset_reset_policy => <<"reset_to_latest">>, offset_commit_interval_seconds => 5 }, - key_encoding_mode => <<"force_utf8">>, + key_encoding_mode => <<"none">>, topic_mapping => [ #{ kafka_topic => <<"kafka-topic-1">>, @@ -124,7 +124,7 @@ values(consumer) -> payload_template => <<"v = ${.value}">> } ], - value_encoding_mode => <<"force_utf8">> + value_encoding_mode => <<"none">> }. %% ------------------------------------------------------------------------------------------------- @@ -334,22 +334,16 @@ fields(consumer_opts) -> #{ required => true, desc => ?DESC(consumer_topic_mapping), - validator => - fun - ([]) -> - {error, "There must be at least one Kafka-MQTT topic mapping"}; - ([_ | _]) -> - ok - end + validator => fun consumer_topic_mapping_validator/1 } )}, {key_encoding_mode, - mk(enum([force_utf8, base64]), #{ - default => force_utf8, desc => ?DESC(consumer_encoding_mode) + mk(enum([none, base64]), #{ + default => none, desc => ?DESC(consumer_encoding_mode) })}, {value_encoding_mode, - mk(enum([force_utf8, base64]), #{ - default => force_utf8, desc => ?DESC(consumer_encoding_mode) + mk(enum([none, base64]), #{ + default => none, desc => ?DESC(consumer_encoding_mode) })} ]; fields(consumer_topic_mapping) -> @@ -449,3 +443,16 @@ kafka_producer_converter( kafka_producer_converter(Config, _HoconOpts) -> %% new schema Config. + +consumer_topic_mapping_validator(_TopicMapping = []) -> + {error, "There must be at least one Kafka-MQTT topic mapping"}; +consumer_topic_mapping_validator(TopicMapping = [_ | _]) -> + NumEntries = length(TopicMapping), + KafkaTopics = [KT || #{<<"kafka_topic">> := KT} <- TopicMapping], + DistinctKafkaTopics = length(lists:usort(KafkaTopics)), + case DistinctKafkaTopics =:= NumEntries of + true -> + ok; + false -> + {error, "Kafka topics must not be repeated in a bridge"} + end. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index 99877ca8e..6abd3ed02 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -61,7 +61,7 @@ }. -type offset_reset_policy() :: reset_to_latest | reset_to_earliest | reset_by_subscriber. %% -type mqtt_payload() :: full_message | message_value. --type encoding_mode() :: force_utf8 | base64. +-type encoding_mode() :: none | base64. -type consumer_init_data() :: #{ hookpoint := binary(), key_encoding_mode := encoding_mode(), @@ -490,7 +490,7 @@ render(FullMessage, PayloadTemplate) -> }, emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, FullMessage, Opts). -encode(Value, force_utf8) -> +encode(Value, none) -> Value; encode(Value, base64) -> base64:encode(Value). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 129011862..cb984fcf6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -576,8 +576,8 @@ kafka_config(TestCase, _KafkaType, Config) -> " offset_reset_policy = reset_to_latest\n" " }\n" "~s" - " key_encoding_mode = force_utf8\n" - " value_encoding_mode = force_utf8\n" + " key_encoding_mode = none\n" + " value_encoding_mode = none\n" " ssl {\n" " enable = ~p\n" " verify = verify_none\n" diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl index 47c21b673..72096c7b1 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl @@ -76,6 +76,68 @@ kafka_producer_test() -> ok. +kafka_consumer_test() -> + Conf1 = parse(kafka_consumer_hocon()), + ?assertMatch( + #{ + <<"bridges">> := + #{ + <<"kafka_consumer">> := + #{ + <<"my_consumer">> := _ + } + } + }, + check(Conf1) + ), + + %% Bad: can't repeat kafka topics. + BadConf1 = emqx_map_lib:deep_put( + [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], + Conf1, + [ + #{ + <<"kafka_topic">> => <<"t1">>, + <<"mqtt_topic">> => <<"mqtt/t1">>, + <<"qos">> => 1, + <<"payload_template">> => <<"${.}">> + }, + #{ + <<"kafka_topic">> => <<"t1">>, + <<"mqtt_topic">> => <<"mqtt/t2">>, + <<"qos">> => 2, + <<"payload_template">> => <<"v = ${.value}">> + } + ] + ), + ?assertThrow( + {_, [ + #{ + path := "bridges.kafka_consumer.my_consumer.topic_mapping", + reason := "Kafka topics must not be repeated in a bridge" + } + ]}, + check(BadConf1) + ), + + %% Bad: there must be at least 1 mapping. + BadConf2 = emqx_map_lib:deep_put( + [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], + Conf1, + [] + ), + ?assertThrow( + {_, [ + #{ + path := "bridges.kafka_consumer.my_consumer.topic_mapping", + reason := "There must be at least one Kafka-MQTT topic mapping" + } + ]}, + check(BadConf2) + ), + + ok. + %%=========================================================================== %% Helper functions %%=========================================================================== @@ -179,3 +241,47 @@ kafka_producer_new_hocon() -> " }\n" "}\n" "". + +%% erlfmt-ignore +kafka_consumer_hocon() -> +""" +bridges.kafka_consumer.my_consumer { + enable = true + bootstrap_hosts = \"kafka-1.emqx.net:9292\" + connect_timeout = 5s + min_metadata_refresh_interval = 3s + metadata_request_timeout = 5s + authentication = { + mechanism = plain + username = emqxuser + password = password + } + kafka { + max_batch_bytes = 896KB + max_rejoin_attempts = 5 + offset_commit_interval_seconds = 3 + offset_reset_policy = reset_to_latest + } + topic_mapping = [ + { + kafka_topic = \"kafka-topic-1\" + mqtt_topic = \"mqtt/topic/1\" + qos = 1 + payload_template = \"${.}\" + }, + { + kafka_topic = \"kafka-topic-2\" + mqtt_topic = \"mqtt/topic/2\" + qos = 2 + payload_template = \"v = ${.value}\" + } + ] + key_encoding_mode = none + value_encoding_mode = none + ssl { + enable = false + verify = verify_none + server_name_indication = \"auto\" + } +} +""". From 19f9c35662793b9e43d1f2dcd369fbf299db6034 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 17:10:29 -0300 Subject: [PATCH 78/88] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- .../i18n/emqx_ee_bridge_kafka.conf | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 2e4a58c7f..a126a0b37 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -537,7 +537,7 @@ emqx_ee_bridge_kafka { consumer_mqtt_payload { desc { en: "The template for transforming the incoming Kafka message." - " By default, it will use JSON format to serialize all visible" + " By default, it will use JSON format to serialize" " inputs from the Kafka message. Such fields are:\n" "headers: an object containing string key-value pairs.\n" "key: Kafka message key (uses the chosen key encoding).\n" @@ -547,17 +547,17 @@ emqx_ee_bridge_kafka { "ts_type: message timestamp type, which is one of" " create, append or undefined.\n" "value: Kafka message value (uses the chosen value encoding).\n" - zh: "用于转换传入的Kafka消息的模板。 " - "默认情况下，它将使用JSON格式来序列化所有来自Kafka消息的可见输入。 " - "这样的字段是。" - "headers: 一个包含字符串键值对的对象。\n" - "key: Kafka消息密钥（使用选择的密钥编码）。\n" - "offset: 信息的偏移量。\n" - "topic: 卡夫卡主题。\n" + zh: "用于转换收到的 Kafka 消息的模板。 " + "默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 " + "这些字段包括：" + "headers：一个包含字符串键值对的 JSON 对象。\n" + "key：Kafka 消息的键（使用选择的编码方式编码）。\n" + "offset：消息的偏移量。\n" + "topic：Kafka 主题。\n" "ts: 消息的时间戳。\n" - "ts_type: 消息的时间戳类型，它是一个" + "ts_type：消息的时间戳类型，值可能是：" " create， append 或 undefined。\n" - "value: Kafka消息值（使用选择的值编码）。\n" + "value: Kafka 消息值（使用选择的编码方式编码）。\n" } label { @@ -589,8 +589,8 @@ emqx_ee_bridge_kafka { } consumer_max_rejoin_attempts { desc { - en: "Maximum number of times allowed for a member to re-join the group." - zh: "允许一个成员重新加入小组的最大次数。" + en: "Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay." + zh: "允许一个成员重新加入小组的最大次数。如果超过改配置次数后仍不能成功加入消费组，则会在延迟一段时间后再重试。" } label { en: "Max Rejoin Attempts" @@ -622,11 +622,11 @@ emqx_ee_bridge_kafka { consumer_topic_mapping { desc { en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." - zh: "定义了Kafka主题和MQTT主题之间的映射。 必须至少包含一个项目。" + zh: "指定 Kafka 主题和 MQTT 主题之间的映射。 必须至少包含一个项目。" } label { en: "Topic Mapping" - zh: "主题图" + zh: "主题映射关系" } } consumer_encoding_mode { From 46c3305ad879b59a3be6260ebc83cb944414548c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 17:12:44 -0300 Subject: [PATCH 79/88] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- changes/ee/feat-9564.en.md | 3 +-- changes/ee/feat-9564.zh.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/changes/ee/feat-9564.en.md b/changes/ee/feat-9564.en.md index 860afbc63..4405e3e07 100644 --- a/changes/ee/feat-9564.en.md +++ b/changes/ee/feat-9564.en.md @@ -1,3 +1,2 @@ Implemented Kafka Consumer bridge. -Now it's possible to consume messages from Kafka and publish them to -MQTT topics. +Now it's possible to consume messages from Kafka and publish them to MQTT topics. diff --git a/changes/ee/feat-9564.zh.md b/changes/ee/feat-9564.zh.md index 0c4867e46..01a7ffe58 100644 --- a/changes/ee/feat-9564.zh.md +++ b/changes/ee/feat-9564.zh.md @@ -1,2 +1,2 @@ 实现了 Kafka 消费者桥接。 -现在可以从Kafka消费消息并将其发布到 MQTT主题。 +现在可以从 Kafka 消费消息并将其发布到 MQTT 主题。 From d464e2aad51eb27b01aee48f9056304172054cce Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 13 Mar 2023 17:13:16 -0300 Subject: [PATCH 80/88] refactor: rename test resource prefix Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_resource/include/emqx_resource.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 283156b95..41be9e8a0 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -119,5 +119,5 @@ -define(AUTO_RESTART_INTERVAL, 60000). -define(AUTO_RESTART_INTERVAL_RAW, <<"60s">>). --define(TEST_ID_PREFIX, "_test-create-dry-run:"). +-define(TEST_ID_PREFIX, "_probe_:"). -define(RES_METRICS, resource_metrics). From 947e01413277c479be33ddaa9d1862b90b890e24 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 14 Mar 2023 15:59:24 -0300 Subject: [PATCH 81/88] test: improve cluster node helpers * Add option to start autocluster. This is useful for scenarios where a cluster is already running and has some configurations set (via config handler/cluster rpc) and later another node joins. * Improve mnesia data directory isolation for nodes. A new dir is set so that we may avoid intra and inter-suite flakiness due to dirty tables and schema. * The janitor is now called synchronously. This ensures the cleanup is done before the test pid dies. --- apps/emqx/test/emqx_common_test_helpers.erl | 36 +++++++++++++++---- apps/emqx/test/emqx_test_janitor.erl | 7 ++++ apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 4 +-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 8d2bd3ba3..3d770d8be 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -67,7 +67,8 @@ emqx_cluster/2, start_epmd/0, start_slave/2, - stop_slave/1 + stop_slave/1, + listener_port/2 ]). -export([clear_screen/0]). @@ -588,6 +589,12 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> %% Whether to execute `emqx_config:init_load(SchemaMod)` %% default: true load_schema => boolean(), + %% If we want to exercise the scenario where a node joins an + %% existing cluster where there has already been some + %% configuration changes (via cluster rpc), then we need to enable + %% autocluster so that the joining node will restart the + %% `emqx_conf' app and correctly catch up the config. + start_autocluster => boolean(), %% Eval by emqx_config:put/2 conf => [{KeyPath :: list(), Val :: term()}], %% Fast option to config listener port @@ -725,9 +732,24 @@ setup_node(Node, Opts) when is_map(Opts) -> %% we need a fresh data dir for each peer node to avoid unintended %% successes due to sharing of data in the cluster. PrivDataDir = maps:get(priv_data_dir, Opts, "/tmp"), + %% If we want to exercise the scenario where a node joins an + %% existing cluster where there has already been some + %% configuration changes (via cluster rpc), then we need to enable + %% autocluster so that the joining node will restart the + %% `emqx_conf' app and correctly catch up the config. + StartAutocluster = maps:get(start_autocluster, Opts, false), %% Load env before doing anything to avoid overriding lists:foreach(fun(App) -> rpc:call(Node, ?MODULE, load, [App]) end, LoadApps), + %% Ensure a clean mnesia directory for each run to avoid + %% inter-test flakiness. + MnesiaDataDir = filename:join([ + PrivDataDir, + node(), + integer_to_list(erlang:unique_integer()), + "mnesia" + ]), + erpc:call(Node, application, set_env, [mnesia, dir, MnesiaDataDir]), %% Needs to be set explicitly because ekka:start() (which calls `gen`) is called without Handler %% in emqx_common_test_helpers:start_apps(...) @@ -792,13 +814,10 @@ setup_node(Node, Opts) when is_map(Opts) -> undefined -> ok; _ -> + StartAutocluster andalso + (ok = rpc:call(Node, emqx_machine_boot, start_autocluster, [])), case rpc:call(Node, ekka, join, [JoinTo]) of ok -> - %% fix cluster rpc, as the conf app is not - %% restarted with the current test procedure. - StartApps andalso - lists:member(emqx_conf, Apps) andalso - (ok = erpc:call(Node, emqx_cluster_rpc, reset, [])), ok; ignore -> ok; @@ -872,6 +891,9 @@ base_port(Number) -> gen_rpc_port(BasePort) -> BasePort - 1. +listener_port(Opts, Type) when is_map(Opts) -> + BasePort = maps:get(base_port, Opts), + listener_port(BasePort, Type); listener_port(BasePort, tcp) -> BasePort; listener_port(BasePort, ssl) -> @@ -1057,7 +1079,7 @@ latency_up_proxy(off, Name, ProxyHost, ProxyPort) -> %% noise in the logs. call_janitor() -> Janitor = get_or_spawn_janitor(), - exit(Janitor, normal), + ok = emqx_test_janitor:stop(Janitor), ok. get_or_spawn_janitor() -> diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index 07d09aca1..703cba6da 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -30,6 +30,7 @@ %% API -export([ start_link/0, + stop/1, push_on_exit_callback/2 ]). @@ -40,6 +41,9 @@ start_link() -> gen_server:start_link(?MODULE, self(), []). +stop(Server) -> + gen_server:call(Server, terminate). + push_on_exit_callback(Server, Callback) when is_function(Callback, 0) -> gen_server:call(Server, {push, Callback}). @@ -56,6 +60,9 @@ terminate(_Reason, #{callbacks := Callbacks}) -> handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) -> {reply, ok, State#{callbacks := [Callback | Callbacks]}}; +handle_call(terminate, _From, State = #{callbacks := Callbacks}) -> + lists:foreach(fun(Fun) -> Fun() end, Callbacks), + {stop, normal, ok, State}; handle_call(_Req, _From, State) -> {reply, error, State}. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index f91233132..260ad1681 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -559,8 +559,8 @@ group_t_copy_plugin_to_a_new_node({'end', Config}) -> ok = rpc:call(CopyToNode, emqx_config, delete_override_conf_files, []), rpc:call(CopyToNode, ekka, leave, []), rpc:call(CopyFromNode, ekka, leave, []), - {ok, _} = emqx_common_test_helpers:stop_slave(CopyToNode), - {ok, _} = emqx_common_test_helpers:stop_slave(CopyFromNode), + ok = emqx_common_test_helpers:stop_slave(CopyToNode), + ok = emqx_common_test_helpers:stop_slave(CopyFromNode), ok = file:del_dir_r(proplists:get_value(to_install_dir, Config)), ok = file:del_dir_r(proplists:get_value(from_install_dir, Config)); group_t_copy_plugin_to_a_new_node(Config) -> From 41b8d4769656fa32d9e6f761c5638c47f1cd35bf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 14 Mar 2023 16:03:48 -0300 Subject: [PATCH 82/88] test(kafka_consumer): add more clusterized tests --- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 403 +++++++++++++++--- 1 file changed, 341 insertions(+), 62 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index cb984fcf6..d72e43963 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -55,6 +55,8 @@ only_once_tests() -> [ t_bridge_rule_action_source, t_cluster_group, + t_node_joins_existing_cluster, + t_cluster_node_down, t_multiple_topic_mappings ]. @@ -924,12 +926,16 @@ action_response(Selected, Envs, Args) -> }), ok. -wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout) -> - do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, #{}). +wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, Timeout) -> + do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, Timeout, #{}). -do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, Acc0) -> - case map_size(Acc0) =:= NPartitions of +do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, Timeout, Acc0) -> + AllPartitionsCovered = map_size(Acc0) =:= NPartitions, + PresentNodes = lists:usort([N || {_Partition, {N, _MemberId}} <- maps:to_list(Acc0)]), + AllNodesCovered = PresentNodes =:= lists:usort(Nodes), + case AllPartitionsCovered andalso AllNodesCovered of true -> + ct:pal("group balanced: ~p", [Acc0]), {ok, Acc0}; false -> receive @@ -942,7 +948,7 @@ do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, Acc0) -> topic_assignments => TopicAssignments }, Acc = reconstruct_assignments_from_events(KafkaTopic, [Event], Acc0), - do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Timeout, Acc) + do_wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, Timeout, Acc) after Timeout -> {timeout, Acc0} end @@ -974,6 +980,123 @@ reconstruct_assignments_from_events(KafkaTopic, Events0, Acc0) -> Assignments ). +setup_group_subscriber_spy(Node) -> + TestPid = self(), + ok = erpc:call( + Node, + fun() -> + ok = meck:new(brod_group_subscriber_v2, [ + passthrough, no_link, no_history, non_strict + ]), + ok = meck:expect( + brod_group_subscriber_v2, + assignments_received, + fun(Pid, MemberId, GenerationId, TopicAssignments) -> + ?tp( + kafka_assignment, + #{ + node => node(), + pid => Pid, + member_id => MemberId, + generation_id => GenerationId, + topic_assignments => TopicAssignments + } + ), + TestPid ! + {kafka_assignment, node(), {Pid, MemberId, GenerationId, TopicAssignments}}, + meck:passthrough([Pid, MemberId, GenerationId, TopicAssignments]) + end + ), + ok + end + ). + +wait_for_cluster_rpc(Node) -> + %% need to wait until the config handler is ready after + %% restarting during the cluster join. + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + true = is_pid(erpc:call(Node, erlang, whereis, [emqx_config_handler])) + ). + +setup_and_start_listeners(Node, NodeOpts) -> + erpc:call( + Node, + fun() -> + lists:foreach( + fun(Type) -> + Port = emqx_common_test_helpers:listener_port(NodeOpts, Type), + ok = emqx_config:put( + [listeners, Type, default, bind], + {{127, 0, 0, 1}, Port} + ), + ok = emqx_config:put_raw( + [listeners, Type, default, bind], + iolist_to_binary([<<"127.0.0.1:">>, integer_to_binary(Port)]) + ), + ok + end, + [tcp, ssl, ws, wss] + ), + ok = emqx_listeners:start(), + ok + end + ). + +cluster(Config) -> + PrivDataDir = ?config(priv_dir, Config), + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, [emqx_conf, emqx_bridge, emqx_rule_engine]}, + {listener_ports, []}, + {peer_mod, slave}, + {priv_data_dir, PrivDataDir}, + {load_schema, true}, + {start_autocluster, true}, + {schema_mod, emqx_ee_conf_schema}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, [broker, router]), + ok; + (emqx_conf) -> + ok; + (_) -> + ok + end} + ] + ), + ct:pal("cluster: ~p", [Cluster]), + Cluster. + +start_async_publisher(Config, KafkaTopic) -> + TId = ets:new(kafka_payloads, [public, ordered_set]), + Loop = fun Go() -> + receive + stop -> ok + after 0 -> + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + publish(Config, KafkaTopic, [#{key => Payload, value => Payload}]), + ets:insert(TId, {Payload}), + timer:sleep(400), + Go() + end + end, + Pid = spawn_link(Loop), + {TId, Pid}. + +stop_async_publisher(Pid) -> + MRef = monitor(process, Pid), + Pid ! stop, + receive + {'DOWN', MRef, process, Pid, _} -> + ok + after 1_000 -> + ct:fail("publisher didn't die") + end, + ok. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -1500,36 +1623,17 @@ t_bridge_rule_action_source(Config) -> ), ok. +%% checks that an existing cluster can be configured with a kafka +%% consumer bridge and that the consumers will distribute over the two +%% nodes. t_cluster_group(Config) -> - ct:timetrap({seconds, 180}), - TestPid = self(), + ct:timetrap({seconds, 150}), NPartitions = ?config(num_partitions, Config), KafkaTopic = ?config(kafka_topic, Config), KafkaName = ?config(kafka_name, Config), ResourceId = resource_id(Config), BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, KafkaName), - PrivDataDir = ?config(priv_dir, Config), - Cluster = emqx_common_test_helpers:emqx_cluster( - [core, core], - [ - {apps, [emqx_conf, emqx_bridge, emqx_rule_engine]}, - {listener_ports, []}, - {peer_mod, slave}, - {priv_data_dir, PrivDataDir}, - {load_schema, true}, - {schema_mod, emqx_ee_conf_schema}, - {env_handler, fun - (emqx) -> - application:set_env(emqx, boot_modules, []), - ok; - (emqx_conf) -> - ok; - (_) -> - ok - end} - ] - ), - ct:pal("cluster: ~p", [Cluster]), + Cluster = cluster(Config), ?check_trace( begin Nodes = @@ -1540,47 +1644,19 @@ t_cluster_group(Config) -> on_exit(fun() -> lists:foreach( fun(N) -> + ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) end, Nodes ) end), - lists:foreach( - fun(N) -> - erpc:call(N, fun() -> - ok = meck:new(brod_group_subscriber_v2, [ - passthrough, no_link, no_history, non_strict - ]), - ok = meck:expect( - brod_group_subscriber_v2, - assignments_received, - fun(Pid, MemberId, GenerationId, TopicAssignments) -> - TestPid ! - {kafka_assignment, node(), - {Pid, MemberId, GenerationId, TopicAssignments}}, - ?tp( - kafka_assignment, - #{ - node => node(), - pid => Pid, - member_id => MemberId, - generation_id => GenerationId, - topic_assignments => TopicAssignments - } - ), - meck:passthrough([Pid, MemberId, GenerationId, TopicAssignments]) - end - ), - ok - end) - end, - Nodes - ), + lists:foreach(fun setup_group_subscriber_spy/1, Nodes), {ok, SRef0} = snabbkaffe:subscribe( ?match_event(#{?snk_kind := kafka_consumer_subscriber_started}), length(Nodes), 15_000 ), + wait_for_cluster_rpc(N2), erpc:call(N2, fun() -> {ok, _} = create_bridge(Config) end), {ok, _} = snabbkaffe:receive_events(SRef0), lists:foreach( @@ -1598,8 +1674,7 @@ t_cluster_group(Config) -> %% sleep so that the two nodes have time to distribute the %% subscribers, rather than just one node containing all %% of them. - ct:sleep(10_000), - {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, 30_000), + {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, 30_000), lists:foreach( fun(N) -> ?assertEqual( @@ -1630,3 +1705,207 @@ t_cluster_group(Config) -> end ), ok. + +%% test that the kafka consumer group rebalances correctly if a bridge +%% already exists when a new EMQX node joins the cluster. +t_node_joins_existing_cluster(Config) -> + ct:timetrap({seconds, 150}), + TopicMapping = ?config(topic_mapping, Config), + [MQTTTopic] = [MQTTTopic || #{mqtt_topic := MQTTTopic} <- TopicMapping], + NPartitions = ?config(num_partitions, Config), + KafkaTopic = ?config(kafka_topic, Config), + KafkaName = ?config(kafka_name, Config), + ResourceId = resource_id(Config), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, KafkaName), + Cluster = cluster(Config), + ?check_trace( + begin + [{Name1, Opts1}, {Name2, Opts2} | _] = Cluster, + N1 = emqx_common_test_helpers:start_slave(Name1, Opts1), + on_exit(fun() -> ok = emqx_common_test_helpers:stop_slave(N1) end), + setup_group_subscriber_spy(N1), + {{ok, _}, {ok, _}} = + ?wait_async_action( + erpc:call(N1, fun() -> {ok, _} = create_bridge(Config) end), + #{?snk_kind := kafka_consumer_subscriber_started}, + 15_000 + ), + ?assertMatch({ok, _}, erpc:call(N1, emqx_bridge, lookup, [BridgeId])), + {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, [N1], 30_000), + ?assertEqual( + {ok, connected}, + erpc:call(N1, emqx_resource_manager, health_check, [ResourceId]) + ), + + %% Now, we start the second node and have it join the cluster. + setup_and_start_listeners(N1, Opts1), + TCPPort1 = emqx_common_test_helpers:listener_port(Opts1, tcp), + {ok, C1} = emqtt:start_link([{port, TCPPort1}, {proto_ver, v5}]), + on_exit(fun() -> catch emqtt:stop(C1) end), + {ok, _} = emqtt:connect(C1), + {ok, _, [2]} = emqtt:subscribe(C1, MQTTTopic, 2), + + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := kafka_consumer_subscriber_started}), + 1, + 30_000 + ), + N2 = emqx_common_test_helpers:start_slave(Name2, Opts2), + on_exit(fun() -> ok = emqx_common_test_helpers:stop_slave(N2) end), + setup_group_subscriber_spy(N2), + Nodes = [N1, N2], + wait_for_cluster_rpc(N2), + + {ok, _} = snabbkaffe:receive_events(SRef0), + ?assertMatch({ok, _}, erpc:call(N2, emqx_bridge, lookup, [BridgeId])), + %% Give some time for the consumers in both nodes to + %% rebalance. + {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, 30_000), + %% Publish some messages so we can check they came from each node. + ?retry( + _Sleep1 = 100, + _Attempts1 = 50, + true = erpc:call(N2, emqx_router, has_routes, [MQTTTopic]) + ), + {ok, SRef1} = + snabbkaffe:subscribe( + ?match_event(#{ + ?snk_kind := kafka_consumer_handle_message, + ?snk_span := {complete, _} + }), + NPartitions, + 10_000 + ), + lists:foreach( + fun(N) -> + Key = <<"k", (integer_to_binary(N))/binary>>, + Val = <<"v", (integer_to_binary(N))/binary>>, + publish(Config, KafkaTopic, [#{key => Key, value => Val}]) + end, + lists:seq(1, NPartitions) + ), + {ok, _} = snabbkaffe:receive_events(SRef1), + + #{nodes => Nodes} + end, + fun(Res, Trace0) -> + #{nodes := Nodes} = Res, + Trace1 = ?of_kind(kafka_assignment, Trace0), + Assignments = reconstruct_assignments_from_events(KafkaTopic, Trace1), + NodeAssignments = lists:usort([ + N + || {_Partition, {N, _MemberId}} <- + maps:to_list(Assignments) + ]), + ?assertEqual(lists:usort(Nodes), NodeAssignments), + ?assertEqual(NPartitions, map_size(Assignments)), + Published = receive_published(#{n => NPartitions, timeout => 3_000}), + ct:pal("published:\n ~p", [Published]), + PublishingNodesFromTrace = + [ + N + || #{ + ?snk_kind := kafka_consumer_handle_message, + ?snk_span := start, + ?snk_meta := #{node := N} + } <- Trace0 + ], + ?assertEqual(lists:usort(Nodes), lists:usort(PublishingNodesFromTrace)), + ok + end + ), + ok. + +%% Checks that the consumers get rebalanced after an EMQX nodes goes +%% down. +t_cluster_node_down(Config) -> + ct:timetrap({seconds, 150}), + TopicMapping = ?config(topic_mapping, Config), + [MQTTTopic] = [MQTTTopic || #{mqtt_topic := MQTTTopic} <- TopicMapping], + NPartitions = ?config(num_partitions, Config), + KafkaTopic = ?config(kafka_topic, Config), + KafkaName = ?config(kafka_name, Config), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, KafkaName), + Cluster = cluster(Config), + ?check_trace( + begin + {_N2, Opts2} = lists:nth(2, Cluster), + Nodes = + [N1, N2 | _] = + lists:map( + fun({Name, Opts}) -> emqx_common_test_helpers:start_slave(Name, Opts) end, + Cluster + ), + on_exit(fun() -> + lists:foreach( + fun(N) -> + ct:pal("stopping ~p", [N]), + ok = emqx_common_test_helpers:stop_slave(N) + end, + Nodes + ) + end), + lists:foreach(fun setup_group_subscriber_spy/1, Nodes), + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := kafka_consumer_subscriber_started}), + length(Nodes), + 15_000 + ), + wait_for_cluster_rpc(N2), + erpc:call(N2, fun() -> {ok, _} = create_bridge(Config) end), + {ok, _} = snabbkaffe:receive_events(SRef0), + lists:foreach( + fun(N) -> + ?assertMatch( + {ok, _}, + erpc:call(N, emqx_bridge, lookup, [BridgeId]), + #{node => N} + ) + end, + Nodes + ), + {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, Nodes, 30_000), + + %% Now, we stop one of the nodes and watch the group + %% rebalance. + setup_and_start_listeners(N2, Opts2), + TCPPort = emqx_common_test_helpers:listener_port(Opts2, tcp), + {ok, C} = emqtt:start_link([{port, TCPPort}, {proto_ver, v5}]), + on_exit(fun() -> catch emqtt:stop(C) end), + {ok, _} = emqtt:connect(C), + {ok, _, [2]} = emqtt:subscribe(C, MQTTTopic, 2), + {TId, Pid} = start_async_publisher(Config, KafkaTopic), + + ct:pal("stopping node ~p", [N1]), + ok = emqx_common_test_helpers:stop_slave(N1), + + %% Give some time for the consumers in remaining node to + %% rebalance. + {ok, _} = wait_until_group_is_balanced(KafkaTopic, NPartitions, [N2], 60_000), + + ok = stop_async_publisher(Pid), + + #{nodes => Nodes, payloads_tid => TId} + end, + fun(Res, Trace0) -> + #{nodes := Nodes, payloads_tid := TId} = Res, + [_N1, N2 | _] = Nodes, + Trace1 = ?of_kind(kafka_assignment, Trace0), + Assignments = reconstruct_assignments_from_events(KafkaTopic, Trace1), + NodeAssignments = lists:usort([ + N + || {_Partition, {N, _MemberId}} <- + maps:to_list(Assignments) + ]), + %% The surviving node has all the partitions assigned to + %% it. + ?assertEqual([N2], NodeAssignments), + ?assertEqual(NPartitions, map_size(Assignments)), + NumPublished = ets:info(TId, size), + %% All published messages are eventually received. + Published = receive_published(#{n => NumPublished, timeout => 3_000}), + ct:pal("published:\n ~p", [Published]), + ok + end + ), + ok. From 0587e0d3f3b6e3b2ee25ce0bfd863bf5f49910bf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 14 Mar 2023 18:16:08 -0300 Subject: [PATCH 83/88] refactor(kafka_consumer): change the generated consumer group id Co-authored-by: Zaiming (Stone) Shi --- .../src/kafka/emqx_bridge_impl_kafka_consumer.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl index 6abd3ed02..44633213c 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl @@ -432,7 +432,7 @@ log_when_error(Fun, Log) -> -spec consumer_group_id(atom() | binary()) -> binary(). consumer_group_id(BridgeName0) -> BridgeName = to_bin(BridgeName0), - <<"emqx-kafka-consumer:", BridgeName/binary>>. + <<"emqx-kafka-consumer-", BridgeName/binary>>. -spec is_dry_run(manager_id()) -> boolean(). is_dry_run(InstanceId) -> From 3954b7bde220a08061b22714f894e5a658ec050e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 15 Mar 2023 15:17:58 -0300 Subject: [PATCH 84/88] fix(bridges): function clause when a non-ingress bridge coexists with an egress bridge This was not caught by our tests because we always test bridge types in isolation. So, if the config only contains ingress-only bridges, the `on_message_publish` hook is never installed. In a real system, if there are bridges of mixed types in the config, the hook might be installed, and `emqx_bridge:get_matched_bridge_id` would crash when iterating over the ingress bridges. --- apps/emqx_bridge/src/emqx_bridge.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d5d5adff1..98ce6a8b0 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -411,7 +411,9 @@ get_matched_bridge_id(BType, Conf, Topic, BName, Acc) when ?EGRESS_DIR_BRIDGES(B do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) end; get_matched_bridge_id(mqtt, #{egress := #{local := #{topic := Filter}}}, Topic, BName, Acc) -> - do_get_matched_bridge_id(Topic, Filter, mqtt, BName, Acc). + do_get_matched_bridge_id(Topic, Filter, mqtt, BName, Acc); +get_matched_bridge_id(_BType, _Conf, _Topic, _BName, Acc) -> + Acc. do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) -> case emqx_topic:match(Topic, Filter) of From 966276127ee8c085c72c9b679f70270488acef47 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 10:52:31 -0300 Subject: [PATCH 85/88] test: trying to make tests more stable --- apps/emqx/test/emqx_common_test_helpers.erl | 8 ++++++-- apps/emqx/test/emqx_test_janitor.erl | 8 ++++++-- .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl | 10 ++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 3d770d8be..38f30b8c5 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -75,7 +75,8 @@ -export([with_mock/4]). -export([ on_exit/1, - call_janitor/0 + call_janitor/0, + call_janitor/1 ]). %% Toxiproxy API @@ -1078,8 +1079,11 @@ latency_up_proxy(off, Name, ProxyHost, ProxyPort) -> %% stop the janitor gracefully to ensure proper cleanup order and less %% noise in the logs. call_janitor() -> + call_janitor(15_000). + +call_janitor(Timeout) -> Janitor = get_or_spawn_janitor(), - ok = emqx_test_janitor:stop(Janitor), + ok = emqx_test_janitor:stop(Janitor, Timeout), ok. get_or_spawn_janitor() -> diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index 703cba6da..c9b297dc7 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -31,6 +31,7 @@ -export([ start_link/0, stop/1, + stop/2, push_on_exit_callback/2 ]). @@ -42,7 +43,10 @@ start_link() -> gen_server:start_link(?MODULE, self(), []). stop(Server) -> - gen_server:call(Server, terminate). + stop(Server, 15_000). + +stop(Server, Timeout) -> + gen_server:call(Server, terminate, Timeout). push_on_exit_callback(Server, Callback) when is_function(Callback, 0) -> gen_server:call(Server, {push, Callback}). @@ -56,7 +60,7 @@ init(Parent) -> {ok, #{callbacks => [], owner => Parent}}. terminate(_Reason, #{callbacks := Callbacks}) -> - lists:foreach(fun(Fun) -> Fun() end, Callbacks). + lists:foreach(fun(Fun) -> catch Fun() end, Callbacks). handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) -> {reply, ok, State#{callbacks := [Callback | Callbacks]}}; diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index d72e43963..15b4fbe40 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -360,7 +360,6 @@ common_init_per_testcase(TestCase, Config0) -> end_per_testcase(_Testcase, Config) -> case proplists:get_bool(skip_does_not_apply, Config) of true -> - ok = snabbkaffe:stop(), ok; false -> ProxyHost = ?config(proxy_host, Config), @@ -1046,12 +1045,19 @@ setup_and_start_listeners(Node, NodeOpts) -> cluster(Config) -> PrivDataDir = ?config(priv_dir, Config), + PeerModule = + case os:getenv("IS_CI") of + false -> + slave; + _ -> + ct_slave + end, Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ {apps, [emqx_conf, emqx_bridge, emqx_rule_engine]}, {listener_ports, []}, - {peer_mod, slave}, + {peer_mod, PeerModule}, {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, From ad1deedd0e04685192b50da5b9a26f1c8f6a4c4a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 16 Mar 2023 21:20:19 +0100 Subject: [PATCH 86/88] build: generate per-lang schema dump --- .github/workflows/build_slim_packages.yaml | 4 ++-- apps/emqx_conf/src/emqx_conf.erl | 22 ++++++++++------------ apps/emqx_dashboard/src/emqx_dashboard.erl | 6 ++++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 08ed8ed2d..163956790 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -82,7 +82,7 @@ jobs: name: "${{ matrix.profile[0] }}_schema_dump" path: | scripts/spellcheck - _build/docgen/${{ matrix.profile[0] }}/schema.json + _build/docgen/${{ matrix.profile[0] }}/schema-en.json windows: runs-on: windows-2019 @@ -218,4 +218,4 @@ jobs: path: /tmp/ - name: Run spellcheck run: | - bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema.json + bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 3da9f0457..c64062861 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -146,17 +146,17 @@ dump_schema(Dir, SchemaModule, I18nFile) -> fun(Lang) -> gen_config_md(Dir, I18nFile, SchemaModule, Lang), gen_api_schema_json(Dir, I18nFile, Lang), - gen_example_conf(Dir, I18nFile, SchemaModule, Lang) + gen_example_conf(Dir, I18nFile, SchemaModule, Lang), + gen_schema_json(Dir, I18nFile, SchemaModule, Lang) end, - [en, zh] - ), - gen_schema_json(Dir, I18nFile, SchemaModule). + ["en", "zh"] + ). %% for scripts/spellcheck. -gen_schema_json(Dir, I18nFile, SchemaModule) -> - SchemaJsonFile = filename:join([Dir, "schema.json"]), +gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> + SchemaJsonFile = filename:join([Dir, "schema-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), - Opts = #{desc_file => I18nFile, lang => "en"}, + Opts = #{desc_file => I18nFile, lang => Lang}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), IoData = jsx:encode(JsonMap, [space, {indent, 4}]), ok = file:write_file(SchemaJsonFile, IoData). @@ -178,17 +178,15 @@ gen_api_schema_json_bridge(Dir, Lang) -> ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo). schema_filename(Dir, Prefix, Lang) -> - Filename = Prefix ++ atom_to_list(Lang) ++ ".json", + Filename = Prefix ++ Lang ++ ".json", filename:join([Dir, Filename]). -gen_config_md(Dir, I18nFile, SchemaModule, Lang0) -> - Lang = atom_to_list(Lang0), +gen_config_md(Dir, I18nFile, SchemaModule, Lang) -> SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]), io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), ok = gen_doc(SchemaMdFile, SchemaModule, I18nFile, Lang). -gen_example_conf(Dir, I18nFile, SchemaModule, Lang0) -> - Lang = atom_to_list(Lang0), +gen_example_conf(Dir, I18nFile, SchemaModule, Lang) -> SchemaMdFile = filename:join([Dir, "emqx.conf." ++ Lang ++ ".example"]), io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), ok = gen_example(SchemaMdFile, SchemaModule, I18nFile, Lang). diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 2afc8b362..060045603 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -132,9 +132,11 @@ stop_listeners(Listeners) -> get_i18n() -> application:get_env(emqx_dashboard, i18n). -init_i18n(File, Lang) -> +init_i18n(File, Lang) when is_atom(Lang) -> + init_i18n(File, atom_to_list(Lang)); +init_i18n(File, Lang) when is_list(Lang) -> Cache = hocon_schema:new_desc_cache(File), - application:set_env(emqx_dashboard, i18n, #{lang => atom_to_binary(Lang), cache => Cache}). + application:set_env(emqx_dashboard, i18n, #{lang => Lang, cache => Cache}). clear_i18n() -> case application:get_env(emqx_dashboard, i18n) of From 3c1254d873089a89271c639eaf4ce48924eb748b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 17 Mar 2023 14:31:43 +0800 Subject: [PATCH 87/88] fix: newly created listeners have no limiter restrictions --- .../src/emqx_limiter/src/emqx_limiter.app.src | 2 +- .../emqx_limiter/src/emqx_limiter_schema.erl | 30 ++++++++++++++----- apps/emqx/src/emqx_schema.erl | 5 +--- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src b/apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src index 69c1c6fb0..4d3dee84e 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter.app.src @@ -2,7 +2,7 @@ {application, emqx_limiter, [ {description, "EMQX Hierarchical Limiter"}, % strict semver, bump manually! - {vsn, "1.0.0"}, + {vsn, "1.0.1"}, {modules, []}, {registered, [emqx_limiter_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index ddfc55f7a..f45fc55b6 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -35,6 +35,12 @@ ]). -define(KILOBYTE, 1024). +-define(BUCKET_KEYS, [ + {bytes_in, bucket_infinity}, + {message_in, bucket_infinity}, + {connection, bucket_limit}, + {message_routing, bucket_infinity} +]). -type limiter_type() :: bytes_in @@ -126,12 +132,18 @@ fields(client_fields) -> })} || Type <- types() ]; -fields(bucket_opts) -> +fields(bucket_infinity) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"infinity">>})}, {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} ]; +fields(bucket_limit) -> + [ + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"1000/s">>})}, + {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"1000">>})}, + {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} + ]; fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, @@ -179,9 +191,9 @@ fields(client_opts) -> )} ]; fields(listener_fields) -> - bucket_fields([bytes_in, message_in, connection, message_routing], listener_client_fields); + bucket_fields(?BUCKET_KEYS, listener_client_fields); fields(listener_client_fields) -> - client_fields([bytes_in, message_in, connection, message_routing]); + client_fields(?BUCKET_KEYS); fields(Type) -> bucket_field(Type). @@ -189,8 +201,10 @@ desc(limiter) -> "Settings for the rate limiter."; desc(node_opts) -> "Settings for the limiter of the node level."; -desc(bucket_opts) -> +desc(bucket_infinity) -> "Settings for the bucket."; +desc(bucket_limit) -> + desc(bucket_infinity); desc(client_opts) -> "Settings for the client in bucket level."; desc(client_fields) -> @@ -337,7 +351,7 @@ apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). bucket_field(Type) when is_atom(Type) -> - fields(bucket_opts) ++ + fields(bucket_infinity) ++ [ {client, ?HOCON( @@ -351,11 +365,11 @@ bucket_field(Type) when is_atom(Type) -> bucket_fields(Types, ClientRef) -> [ {Type, - ?HOCON(?R_REF(?MODULE, bucket_opts), #{ + ?HOCON(?R_REF(?MODULE, Opts), #{ desc => ?DESC(?MODULE, Type), required => false })} - || Type <- Types + || {Type, Opts} <- Types ] ++ [ {client, @@ -375,5 +389,5 @@ client_fields(Types) -> desc => ?DESC(Type), required => false })} - || Type <- Types + || {Type, _} <- Types ]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 275a9592e..b18534a42 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1903,10 +1903,7 @@ base_listener(Bind) -> listener_fields ), #{ - desc => ?DESC(base_listener_limiter), - default => #{ - <<"connection">> => #{<<"rate">> => <<"1000/s">>, <<"capacity">> => 1000} - } + desc => ?DESC(base_listener_limiter) } )}, {"enable_authn", From c47c9feff1bb1f6147e46f609ef595a2da9c27f5 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 17 Mar 2023 16:23:31 +0800 Subject: [PATCH 88/88] chore: upgrade dashboard to e1.0.5-beta.1 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 370c861d6..e49621615 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.1.9 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.4 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5-beta.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT)