Merge pull request #10005 from id/ci-try-to-fix-flaky-kafka

ci: fix flaky kafka tests

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -39,9 +39,12 @@ services:
     container_name: kafka-1.emqx.net
     hostname: kafka-1.emqx.net
     depends_on:
-      - "kdc"
+      kdc:
-      - "zookeeper"
+        condition: service_started
-      - "ssl_cert_gen"
+      zookeeper:
+        condition: service_started
+      ssl_cert_gen:
+        condition: service_completed_successfully
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
@@ -52,7 +55,7 @@ services:
       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
       KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
-      KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
+      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer

lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl

@@ -277,6 +277,9 @@ kafka_bridge_rest_api_helper(Config) ->
             },
             <<"kafka">> => #{
                 <<"topic">> => erlang:list_to_binary(KafkaTopic),
+                <<"buffer">> => #{
+                    <<"memory_overload_protection">> => <<"false">>
+                },
                 <<"message">> => #{
                     <<"key">> => <<"${clientid}">>,
                     <<"value">> => <<"${.payload}">>
@@ -384,6 +387,13 @@ t_failed_creation_then_fix(Config) ->
         "kafka_hosts_string" => HostsString,
         "kafka_topic" => KafkaTopic,
         "instance_id" => ResourceId,
+        "producer" => #{
+            "kafka" => #{
+                "buffer" => #{
+                    "memory_overload_protection" => false
+                }
+            }
+        },
         "ssl" => #{}
     }),
     %% creates, but fails to start producers
@@ -577,6 +587,9 @@ producer = {
         topic = \"{{ kafka_topic }}\"
         message = {key = \"${clientid}\", value = \"${.payload}\"}
         partition_strategy = {{ partition_strategy }}
+        buffer = {
+            memory_overload_protection = false
+        }
     }
 }
 """.

scripts/ct/run.sh

@@ -201,7 +201,7 @@ if [ "$STOP" = 'no' ]; then
     # some left-over log file has to be deleted before a new docker-compose up
     rm -f '.ci/docker-compose-file/redis/*.log'
     # shellcheck disable=2086 # no quotes for F_OPTIONS
-    docker-compose $F_OPTIONS up -d --build --remove-orphans
+    docker compose $F_OPTIONS up -d --build --remove-orphans
 fi
 
 echo "Fixing file owners and permissions for $UID_GID"
@@ -218,7 +218,7 @@ set +e
 
 if [ "$STOP" = 'yes' ]; then
     # shellcheck disable=2086 # no quotes for F_OPTIONS
-    docker-compose $F_OPTIONS down --remove-orphans
+    docker compose $F_OPTIONS down --remove-orphans
 elif [ "$ATTACH" = 'yes' ]; then
     docker exec -it "$ERLANG_CONTAINER" bash
 elif [ "$CONSOLE" = 'yes' ]; then
@@ -235,11 +235,11 @@ else
         LOG='_build/test/logs/docker-compose.log'
         echo "Dumping docker-compose log to $LOG"
         # shellcheck disable=2086 # no quotes for F_OPTIONS
-        docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
+        docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
     fi
     if [ "$KEEP_UP" != 'yes' ]; then
         # shellcheck disable=2086 # no quotes for F_OPTIONS
-        docker-compose $F_OPTIONS down
+        docker compose $F_OPTIONS down
     fi
     exit $RESULT
 fi