From 2790d5697d8e1c323b5b4c71f18c2a08c45fa6ce Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 28 Dec 2022 18:48:53 +0100 Subject: [PATCH 1/6] test: add a 2 seconds sleep between listener sotp and restart to lower the chance of eaddrinuse --- apps/emqx/test/emqx_listeners_SUITE.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 5eb216be5..1b503b24e 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -133,6 +133,8 @@ t_start_stop_listeners(_) -> t_restart_listeners(_) -> ok = emqx_listeners:start(), ok = emqx_listeners:stop(), + %% flakyness: eaddrinuse + timer:sleep(timer:seconds(2)), ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). From 0b43ae621d9c40cfce512062cbc25cf857091e3b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 28 Dec 2022 23:06:47 +0100 Subject: [PATCH 2/6] ci: dump docker-compose log if failed to run ct --- .../test/emqx_bridge_mqtt_SUITE.erl | 2 +- scripts/ct/run.sh | 30 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index 1bf156ed4..9aba5715a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -302,7 +302,7 @@ t_egress_custom_clientid_prefix(_Config) -> receive {deliver, RemoteTopic, #message{from = From}} -> Size = byte_size(ResourceID), - ?assertMatch(<<"my-custom-prefix:", ResouceID:Size/binary, _/binary>>, From), + ?assertMatch(<<"my-custom-prefix:", _ResouceID:Size/binary, _/binary>>, From), ok after 1000 -> ct:fail("should have published message") diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index d86ee4f6a..1331adbf5 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -12,7 +12,7 @@ help() { echo "-h|--help: To display this usage info" echo "--app lib_dir/app_name: For which app to run start docker-compose, and run common tests" echo "--suites SUITE1,SUITE2: Comma separated SUITE names to run. e.g. apps/emqx/test/emqx_SUITE.erl" - echo "--console: Start EMQX in console mode" + echo "--console: Start EMQX in console mode but do not run test cases" echo "--attach: Attach to the Erlang docker container without running any test case" echo "--only-up: Only start the testbed but do not run CT" echo "--keep-up: Keep the testbed running after CT" @@ -143,7 +143,7 @@ F_OPTIONS="" for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done - +ORIG_UID_GID="$UID:$UID" if [[ "${NEED_ROOT:-}" == 'yes' ]]; then export UID_GID='root:root' else @@ -152,7 +152,7 @@ else # Permissions issue happens because we are mounting local filesystem # where files are owned by $UID to docker container where it's using # root (UID=0) by default, and git is not happy about it. - export UID_GID="$UID:$UID" + export UID_GID="$ORIG_UID_GID" fi # shellcheck disable=2086 # no quotes for F_OPTIONS @@ -171,23 +171,37 @@ docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache & # need to initialize .erlang.cookie manually here because / is not writable by $UID docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie" +restore_ownership() { + if [[ "$ORIG_UID_GID" != "$UID_GID" ]]; then + docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $ORIG_UID_GID /emqx" + fi +} + if [ "$ONLY_UP" = 'yes' ]; then exit 0 fi +set +e + if [ "$ATTACH" = 'yes' ]; then docker exec -it "$ERLANG_CONTAINER" bash + restore_ownership elif [ "$CONSOLE" = 'yes' ]; then docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run" + restore_ownership else - set +e docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" RESULT=$? - if [ "$KEEP_UP" = 'yes' ]; then - exit $RESULT - else + restore_ownership + if [ $RESULT -ne 0 ]; then + LOG='_build/test/logs/docker-compose.log' + echo "Dumping docker-compose log to $LOG" + # shellcheck disable=2086 # no quotes for F_OPTIONS + docker-compose $F_OPTIONS logs --no-color > "$LOG" + fi + if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS docker-compose $F_OPTIONS down - exit $RESULT fi + exit $RESULT fi From 5ae4bd672268aabde0d53ded5d0df58d51845bdd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 29 Dec 2022 10:29:36 +0100 Subject: [PATCH 3/6] chore: bump influxdb client lib version from 1.1.4 to 1.1.5 fixed a compile warning --- lib-ee/emqx_ee_connector/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 1419c2070..ab4c88396 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.4"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.5"}}}, {emqx, {path, "../../apps/emqx"}} ]}. From 9f346b44ee6b3f1e1a43fbf7645b453087028f2e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 29 Dec 2022 10:36:16 +0100 Subject: [PATCH 4/6] ci: avoid github action ct log artifacts name clash --- .github/workflows/run_test_cases.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 4ee8efa2e..e8534ef68 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -170,7 +170,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.profile }}-${{ matrix.prefix }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }} path: source/_build/test/logs ct: @@ -213,7 +213,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.profile }}-${{ matrix.prefix }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }} path: source/_build/test/logs make_cover: From e69e016e54ea1af19293618ffb08f3b87f8789f8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 29 Dec 2022 10:56:14 +0100 Subject: [PATCH 5/6] ci: show timestamp from docker compose logs --- scripts/ct/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 1331adbf5..756f1520f 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -197,7 +197,7 @@ else LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS logs --no-color > "$LOG" + docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG" fi if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS From 6d7e6e6ea764ccd2d44a065e077b76f3bc924108 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 29 Dec 2022 14:03:28 +0100 Subject: [PATCH 6/6] ci: delay Kafka topic creation --- .ci/docker-compose-file/docker-compose-kafka.yaml | 6 +++--- .../kafka/{run_add_scram_users.sh => kafka-entrypoint.sh} | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) rename .ci/docker-compose-file/kafka/{run_add_scram_users.sh => kafka-entrypoint.sh} (83%) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 716c4c709..9662b174d 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -54,7 +54,7 @@ services: KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, + KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks KAFKA_SSL_TRUSTSTORE_PASSWORD: password @@ -66,8 +66,8 @@ services: volumes: - emqx-shared-secret:/var/lib/secret - ./kafka/jaas.conf:/etc/kafka/jaas.conf - - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh + - ./kafka/kafka-entrypoint.sh:/bin/kafka-entrypoint.sh - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf - command: run_add_scram_users.sh + command: kafka-entrypoint.sh diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/kafka-entrypoint.sh similarity index 83% rename from .ci/docker-compose-file/kafka/run_add_scram_users.sh rename to .ci/docker-compose-file/kafka/kafka-entrypoint.sh index 4b51fee0d..445fd65c9 100755 --- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh +++ b/.ci/docker-compose-file/kafka/kafka-entrypoint.sh @@ -22,6 +22,7 @@ sleep 3 echo "+++++++ Starting Kafka ++++++++" +# fork start Kafka start-kafka.sh & SERVER=localhost @@ -41,6 +42,12 @@ echo "+++++++ Run config commands ++++++++" kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=password],SCRAM-SHA-512=[password=password]' --entity-type users --entity-name emqxuser +echo "+++++++ Creating Kafka Topics ++++++++" + +# create topics after re-configuration +# there seem to be a race condition when creating the topics (too early) +env KAFKA_CREATE_TOPICS="$KAFKA_CREATE_TOPICS_NG" KAFKA_PORT="$PORT1" create-topics.sh + echo "+++++++ Wait until Kafka ports are down ++++++++" bash -c 'while printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $SERVER $PORT1