From fc12a8c4c8a1894d1532d13d50468852681a28b4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 5 Jan 2023 07:22:21 +0100 Subject: [PATCH 1/4] test: do not use tc_user_skip for test cases --- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 91fd2f399..0eadf46ad 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -110,7 +110,7 @@ init_per_testcase(TestCase, Config0) when -> case ?config(batch_size, Config0) of 1 -> - {skip, no_batching}; + [{skip_due_to_no_batching, true}]; _ -> {ok, _} = start_echo_http_server(), delete_all_bridges(), @@ -754,6 +754,15 @@ t_publish_templated(Config) -> ok. t_publish_success_batch(Config) -> + case proplists:get_bool(skip_due_to_no_batching, Config) of + true -> + ct:pal("this test case is skipped due to non-applicable config"), + ok; + false -> + test_publish_success_batch(Config) + end. + +test_publish_success_batch(Config) -> ResourceId = ?config(resource_id, Config), ServiceAccountJSON = ?config(service_account_json, Config), TelemetryTable = ?config(telemetry_table, Config), From 5f12cdff6c3e35cad220ceabd427b3eec41a6e32 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 3 Jan 2023 10:53:10 +0100 Subject: [PATCH 2/4] ci: should not skip tests in github actions --- .github/workflows/run_test_cases.yaml | 2 +- .../test/emqx_connector_redis_SUITE.erl | 9 +++++++- .../test/emqx_ee_bridge_redis_SUITE.erl | 8 +++++++ scripts/ct/run.sh | 21 +++++++++++++++---- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index e8534ef68..91a087c95 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -162,7 +162,7 @@ jobs: INFLUXDB_TAG: 2.5.0 PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} - run: ./scripts/ct/run.sh --app ${{ matrix.app }} + run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} - uses: actions/upload-artifact@v3 with: name: coverdata diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index fa742da0b..5584af763 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -50,7 +50,7 @@ init_per_suite(Config) -> {ok, _} = application:ensure_all_started(emqx_connector), Config; false -> - {skip, no_redis} + assert_ci() end. end_per_suite(_Config) -> @@ -63,6 +63,13 @@ init_per_testcase(_, Config) -> end_per_testcase(_, _Config) -> ok. +assert_ci() -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_redis); + _ -> + {skip, no_redis} + end. % %%------------------------------------------------------------------------------ % %% Testcases % %%------------------------------------------------------------------------------ diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index f88bc42eb..1f4b52ddc 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -96,6 +96,14 @@ init_per_suite(Config) -> | Config ]; false -> + assert_ci() + end. + +assert_ci() -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_redis); + _ -> {skip, no_redis} end. diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 0a64794b6..fbe3b7f64 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -17,6 +17,9 @@ help() { echo "--stop: Stop running containers for the given app" echo "--only-up: Only start the testbed but do not run CT" echo "--keep-up: Keep the testbed running after CT" + echo "--ci: Set this flag in GitHub action to enforce no tests are skipped" + echo "--" If any, all args after '--' are passed to rebar3 ct + echo " otherwise it runs the entire app's CT" } WHICH_APP='novalue' @@ -26,6 +29,7 @@ ONLY_UP='no' SUITES='' ATTACH='no' STOP='no' +IS_CI='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -56,9 +60,14 @@ while [ "$#" -gt 0 ]; do CONSOLE='yes' shift 1 ;; - --suites) - SUITES="$2" - shift 2 + --ci) + IS_CI='yes' + shift 1 + ;; + --) + shift 1 + REBAR3CT="$*" + shift $# ;; *) echo "unknown option $1" @@ -202,7 +211,11 @@ elif [ "$CONSOLE" = 'yes' ]; then docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run" restore_ownership else - docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" + if [ -z "${REBAR3CT:-}" ]; then + docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" + else + docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "./rebar3 ct $REBAR3CT" + fi RESULT=$? restore_ownership if [ $RESULT -ne 0 ]; then From 7a4e7a6de9c274e10e942fe7ab83ed46f4b92684 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 5 Jan 2023 10:48:37 +0100 Subject: [PATCH 3/4] test: wait for redis --- .../test/emqx_connector_redis_SUITE.erl | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index 5584af763..060340fed 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -36,22 +36,16 @@ groups() -> []. init_per_suite(Config) -> - case - emqx_common_test_helpers:is_all_tcp_servers_available( - [ - {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT}, - {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT} - ] - ) - of - true -> - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - Config; - false -> - assert_ci() - end. + Checks = + case os:getenv("IS_CI") of + "yes" -> 10; + _ -> 1 + end, + ok = wait_for_redis(Checks), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + Config. end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_resource]), @@ -63,13 +57,24 @@ init_per_testcase(_, Config) -> end_per_testcase(_, _Config) -> ok. -assert_ci() -> - case os:getenv("IS_CI") of - "yes" -> - throw(no_redis); - _ -> - {skip, no_redis} +wait_for_redis(0) -> + throw(timeout); +wait_for_redis(Checks) -> + case + emqx_common_test_helpers:is_all_tcp_servers_available( + [ + {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT}, + {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT} + ] + ) + of + true -> + ok; + false -> + timer:sleep(1000), + wait_for_redis(Checks - 1) end. + % %%------------------------------------------------------------------------------ % %% Testcases % %%------------------------------------------------------------------------------ From a23fdcab6f19f9528b6a9477243566656cc2bc42 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 5 Jan 2023 22:51:11 +0100 Subject: [PATCH 4/4] chore: delete unused var in ct/run.sh --- scripts/ct/run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index fbe3b7f64..7fb9e00fd 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -26,7 +26,6 @@ WHICH_APP='novalue' CONSOLE='no' KEEP_UP='no' ONLY_UP='no' -SUITES='' ATTACH='no' STOP='no' IS_CI='no'