Merge pull request #9629 from zmstone/1128-fix-flaky-test-listener-restart-eaddrinuse

test: add a 2 seconds sleep between listener sotp and restart

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -54,7 +54,7 @@ services:
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
-      KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
+      KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
       KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks
       KAFKA_SSL_TRUSTSTORE_PASSWORD: password
@@ -66,8 +66,8 @@ services:
     volumes:
       - emqx-shared-secret:/var/lib/secret
       - ./kafka/jaas.conf:/etc/kafka/jaas.conf
-      - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
+      - ./kafka/kafka-entrypoint.sh:/bin/kafka-entrypoint.sh
       - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
       - ./kerberos/krb5.conf:/etc/krb5.conf
-    command: run_add_scram_users.sh
+    command: kafka-entrypoint.sh
 

.ci/docker-compose-file/kafka/kafka-entrypoint.sh

@@ -22,6 +22,7 @@ sleep 3
 
 echo "+++++++ Starting Kafka ++++++++"
 
+# fork start Kafka
 start-kafka.sh &
 
 SERVER=localhost
@@ -41,6 +42,12 @@ echo "+++++++ Run config commands ++++++++"
 
 kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=password],SCRAM-SHA-512=[password=password]' --entity-type users --entity-name emqxuser
 
+echo "+++++++ Creating Kafka Topics ++++++++"
+
+# create topics after re-configuration
+# there seem to be a race condition when creating the topics (too early)
+env KAFKA_CREATE_TOPICS="$KAFKA_CREATE_TOPICS_NG" KAFKA_PORT="$PORT1" create-topics.sh
+
 echo "+++++++ Wait until Kafka ports are down ++++++++"
 
 bash -c 'while printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1

.github/workflows/run_test_cases.yaml

@@ -170,7 +170,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}
+            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
             path: source/_build/test/logs
 
     ct:
@@ -213,7 +213,7 @@ jobs:
         - uses: actions/upload-artifact@v3
           if: failure()
           with:
-            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}
+            name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
             path: source/_build/test/logs
 
     make_cover:

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -133,6 +133,8 @@ t_start_stop_listeners(_) ->
 t_restart_listeners(_) ->
     ok = emqx_listeners:start(),
     ok = emqx_listeners:stop(),
+    %% flakyness: eaddrinuse
+    timer:sleep(timer:seconds(2)),
     ok = emqx_listeners:restart(),
     ok = emqx_listeners:stop().
 

apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl

@@ -302,7 +302,7 @@ t_egress_custom_clientid_prefix(_Config) ->
     receive
         {deliver, RemoteTopic, #message{from = From}} ->
             Size = byte_size(ResourceID),
-            ?assertMatch(<<"my-custom-prefix:", ResouceID:Size/binary, _/binary>>, From),
+            ?assertMatch(<<"my-custom-prefix:", _ResouceID:Size/binary, _/binary>>, From),
             ok
     after 1000 ->
         ct:fail("should have published message")

lib-ee/emqx_ee_connector/rebar.config

@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
-  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}},
+  {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.5"}}},
   {emqx, {path, "../../apps/emqx"}}
 ]}.
 

scripts/ct/run.sh

@@ -12,7 +12,7 @@ help() {
     echo "-h|--help:              To display this usage info"
     echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests"
     echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl"
-    echo "--console:              Start EMQX in console mode"
+    echo "--console:              Start EMQX in console mode but do not run test cases"
     echo "--attach:               Attach to the Erlang docker container without running any test case"
     echo "--only-up:              Only start the testbed but do not run CT"
     echo "--keep-up:              Keep the testbed running after CT"
@@ -143,7 +143,7 @@ F_OPTIONS=""
 for file in "${FILES[@]}"; do
     F_OPTIONS="$F_OPTIONS -f $file"
 done
-
+ORIG_UID_GID="$UID:$UID"
 if [[ "${NEED_ROOT:-}" == 'yes' ]]; then
     export UID_GID='root:root'
 else
@@ -152,7 +152,7 @@ else
     # Permissions issue happens because we are mounting local filesystem
     # where files are owned by $UID to docker container where it's using
     # root (UID=0) by default, and git is not happy about it.
-    export UID_GID="$UID:$UID"
+    export UID_GID="$ORIG_UID_GID"
 fi
 
 # shellcheck disable=2086 # no quotes for F_OPTIONS
@@ -171,23 +171,37 @@ docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache &
 # need to initialize .erlang.cookie manually here because / is not writable by $UID
 docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie"
 
+restore_ownership() {
+    if [[ "$ORIG_UID_GID" != "$UID_GID" ]]; then
+        docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $ORIG_UID_GID /emqx"
+    fi
+}
+
 if [ "$ONLY_UP" = 'yes' ]; then
     exit 0
 fi
 
+set +e
+
 if [ "$ATTACH" = 'yes' ]; then
     docker exec -it "$ERLANG_CONTAINER" bash
+    restore_ownership
 elif [ "$CONSOLE" = 'yes' ]; then
     docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
+    restore_ownership
 else
-    set +e
     docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
     RESULT=$?
-    if [ "$KEEP_UP" = 'yes' ]; then
-        exit $RESULT
-    else
+    restore_ownership
+    if [ $RESULT -ne 0 ]; then
+        LOG='_build/test/logs/docker-compose.log'
+        echo "Dumping docker-compose log to $LOG"
+        # shellcheck disable=2086 # no quotes for F_OPTIONS
+        docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
+    fi
+    if [ "$KEEP_UP" != 'yes' ]; then
         # shellcheck disable=2086 # no quotes for F_OPTIONS
         docker-compose $F_OPTIONS down
-        exit $RESULT
     fi
+    exit $RESULT
 fi