From 7c917334dcb17d6724f8278842981eae7fecc07a Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 16:21:18 +0100 Subject: [PATCH 1/5] ci: enforce stricter condition on ssl_cert_gen dependency --- .ci/docker-compose-file/docker-compose-kafka.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index e3ade50a1..4c2d2018e 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -39,9 +39,12 @@ services: container_name: kafka-1.emqx.net hostname: kafka-1.emqx.net depends_on: - - "kdc" - - "zookeeper" - - "ssl_cert_gen" + kdc: + condition: service_started + zookeeper: + condition: service_started + ssl_cert_gen: + condition: service_completed_successfully environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 From d5174c1555b0370f923737c6b473c0cdeea129e6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 16:47:21 +0100 Subject: [PATCH 2/5] ci: make sure we use latest compose plugin https://docs.docker.com/compose/#compose-v2-and-the-new-docker-compose-command --- scripts/ct/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 372f5ca11..b44095624 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -201,7 +201,7 @@ if [ "$STOP" = 'no' ]; then # some left-over log file has to be deleted before a new docker-compose up rm -f '.ci/docker-compose-file/redis/*.log' # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS up -d --build --remove-orphans + docker compose $F_OPTIONS up -d --build --remove-orphans fi echo "Fixing file owners and permissions for $UID_GID" @@ -218,7 +218,7 @@ set +e if [ "$STOP" = 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down --remove-orphans + docker compose $F_OPTIONS down --remove-orphans elif [ "$ATTACH" = 'yes' ]; then docker exec -it "$ERLANG_CONTAINER" bash elif [ "$CONSOLE" = 'yes' ]; then @@ -235,11 +235,11 @@ else LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG" + docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG" fi if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down + docker compose $F_OPTIONS down fi exit $RESULT fi From 977bfb3d6c8af2254af59d9596953422f793ecdd Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 16:47:50 +0100 Subject: [PATCH 3/5] ci(kafka): use more suitable env variable to pass jaas conf --- .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 4c2d2018e..976b0bc1c 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -55,7 +55,7 @@ services: KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN - KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer From 9d30a529108d614427c350fe3d6d4d7524095362 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 19:29:26 +0100 Subject: [PATCH 4/5] test: disable replayq memory overload protection --- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 17484b948..e06018c39 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -577,6 +577,9 @@ producer = { topic = \"{{ kafka_topic }}\" message = {key = \"${clientid}\", value = \"${.payload}\"} partition_strategy = {{ partition_strategy }} + buffer = { + memory_overload_protection = false + } } } """. From c869eff6e8ab7705006740af0a0921d691661a08 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 20 Feb 2023 21:38:14 +0100 Subject: [PATCH 5/5] test(kafka): disable overload protection in 2 more places --- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index e06018c39..d06218397 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -277,6 +277,9 @@ kafka_bridge_rest_api_helper(Config) -> }, <<"kafka">> => #{ <<"topic">> => erlang:list_to_binary(KafkaTopic), + <<"buffer">> => #{ + <<"memory_overload_protection">> => <<"false">> + }, <<"message">> => #{ <<"key">> => <<"${clientid}">>, <<"value">> => <<"${.payload}">> @@ -384,6 +387,13 @@ t_failed_creation_then_fix(Config) -> "kafka_hosts_string" => HostsString, "kafka_topic" => KafkaTopic, "instance_id" => ResourceId, + "producer" => #{ + "kafka" => #{ + "buffer" => #{ + "memory_overload_protection" => false + } + } + }, "ssl" => #{} }), %% creates, but fails to start producers