Merge pull request #9684 from zmstone/0105-do-not-use-testcase-skip

test: do not use tc_user_skip for test cases

.github/workflows/run_test_cases.yaml

@@ -162,7 +162,7 @@ jobs:
             INFLUXDB_TAG: 2.5.0
             PROFILE: ${{ matrix.profile }}
             CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
-          run: ./scripts/ct/run.sh --app ${{ matrix.app }}
+          run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
         - uses: actions/upload-artifact@v3
           with:
             name: coverdata

apps/emqx_connector/test/emqx_connector_redis_SUITE.erl

@@ -36,22 +36,16 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    case
+    Checks =
-        emqx_common_test_helpers:is_all_tcp_servers_available(
+        case os:getenv("IS_CI") of
-            [
+            "yes" -> 10;
-                {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT},
+            _ -> 1
-                {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT}
+        end,
-            ]
+    ok = wait_for_redis(Checks),
-        )
+    ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-    of
+    ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
-        true ->
+    {ok, _} = application:ensure_all_started(emqx_connector),
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+    Config.
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
-            {ok, _} = application:ensure_all_started(emqx_connector),
-            Config;
-        false ->
-            {skip, no_redis}
-    end.
 
 end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_resource]),
@@ -63,6 +57,24 @@ init_per_testcase(_, Config) ->
 end_per_testcase(_, _Config) ->
     ok.
 
+wait_for_redis(0) ->
+    throw(timeout);
+wait_for_redis(Checks) ->
+    case
+        emqx_common_test_helpers:is_all_tcp_servers_available(
+            [
+                {?REDIS_SINGLE_HOST, ?REDIS_SINGLE_PORT},
+                {?REDIS_SENTINEL_HOST, ?REDIS_SENTINEL_PORT}
+            ]
+        )
+    of
+        true ->
+            ok;
+        false ->
+            timer:sleep(1000),
+            wait_for_redis(Checks - 1)
+    end.
+
 % %%------------------------------------------------------------------------------
 % %% Testcases
 % %%------------------------------------------------------------------------------

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl

@@ -110,7 +110,7 @@ init_per_testcase(TestCase, Config0) when
 ->
     case ?config(batch_size, Config0) of
         1 ->
-            {skip, no_batching};
+            [{skip_due_to_no_batching, true}];
         _ ->
             {ok, _} = start_echo_http_server(),
             delete_all_bridges(),
@@ -754,6 +754,15 @@ t_publish_templated(Config) ->
     ok.
 
 t_publish_success_batch(Config) ->
+    case proplists:get_bool(skip_due_to_no_batching, Config) of
+        true ->
+            ct:pal("this test case is skipped due to non-applicable config"),
+            ok;
+        false ->
+            test_publish_success_batch(Config)
+    end.
+
+test_publish_success_batch(Config) ->
     ResourceId = ?config(resource_id, Config),
     ServiceAccountJSON = ?config(service_account_json, Config),
     TelemetryTable = ?config(telemetry_table, Config),

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl

@@ -96,6 +96,14 @@ init_per_suite(Config) ->
                 | Config
             ];
         false ->
+            assert_ci()
+    end.
+
+assert_ci() ->
+    case os:getenv("IS_CI") of
+        "yes" ->
+            throw(no_redis);
+        _ ->
             {skip, no_redis}
     end.
 

scripts/ct/run.sh

@@ -17,15 +17,18 @@ help() {
     echo "--stop:                 Stop running containers for the given app"
     echo "--only-up:              Only start the testbed but do not run CT"
     echo "--keep-up:              Keep the testbed running after CT"
+    echo "--ci:                   Set this flag in GitHub action to enforce no tests are skipped"
+    echo "--"                     If any, all args after '--' are passed to rebar3 ct
+    echo "                        otherwise it runs the entire app's CT"
 }
 
 WHICH_APP='novalue'
 CONSOLE='no'
 KEEP_UP='no'
 ONLY_UP='no'
-SUITES=''
 ATTACH='no'
 STOP='no'
+IS_CI='no'
 while [ "$#" -gt 0 ]; do
     case $1 in
         -h|--help)
@@ -56,9 +59,14 @@ while [ "$#" -gt 0 ]; do
             CONSOLE='yes'
             shift 1
             ;;
-        --suites)
+        --ci)
-            SUITES="$2"
+            IS_CI='yes'
-            shift 2
+            shift 1
+            ;;
+        --)
+            shift 1
+            REBAR3CT="$*"
+            shift $#
             ;;
         *)
             echo "unknown option $1"
@@ -202,7 +210,11 @@ elif [ "$CONSOLE" = 'yes' ]; then
     docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
     restore_ownership
 else
-    docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
+    if [ -z "${REBAR3CT:-}" ]; then
+        docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
+    else
+        docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "./rebar3 ct $REBAR3CT"
+    fi
     RESULT=$?
     restore_ownership
     if [ $RESULT -ne 0 ]; then