From 045875d18d233100bcc7aeec1bca7dac2b9e3cdf Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 30 Oct 2023 10:32:27 +0100 Subject: [PATCH] ci: make CI a little faster and less flaky - use latest ubuntu22.04 emqx-builder image in compose files - use xl runners for compiling and for emqx app tests - make helm tests less flaky --- .../docker-compose-kafka.yaml | 2 +- .ci/docker-compose-file/docker-compose.yaml | 16 ++-- .github/workflows/_pr_entrypoint.yaml | 5 +- .github/workflows/run_helm_tests.yaml | 7 +- .github/workflows/run_test_cases.yaml | 4 +- Makefile | 6 ++ scripts/ct/run.sh | 67 +++++++++------ scripts/find-apps.sh | 81 ++++++++++--------- 8 files changed, 109 insertions(+), 79 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index b39526686..352494592 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -18,7 +18,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04 container_name: kdc.emqx.net expose: - 88 # kdc diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 9adbef02e..d4a44bfb0 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,17 +3,17 @@ version: '3.9' services: erlang: container_name: erlang - image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04} + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04} env_file: - conf.env environment: - GITHUB_ACTIONS: ${GITHUB_ACTIONS} - GITHUB_TOKEN: ${GITHUB_TOKEN} - GITHUB_RUN_ID: ${GITHUB_RUN_ID} - GITHUB_SHA: ${GITHUB_SHA} - GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER} - GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME} - GITHUB_REF: ${GITHUB_REF} + GITHUB_ACTIONS: ${GITHUB_ACTIONS:-} + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + GITHUB_RUN_ID: ${GITHUB_RUN_ID:-} + GITHUB_SHA: ${GITHUB_SHA:-} + GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER:-} + GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME:-} + GITHUB_REF: ${GITHUB_REF:-} networks: - emqx_bridge ports: diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index d1f5b252f..3e3db875b 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -25,6 +25,7 @@ jobs: version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted","ephemeral","linux","x64"]' || '["ubuntu-22.04"]' }} + xl_runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted","ephemeral-xl","linux","x64"]' || '["ubuntu-22.04"]' }} builder: "ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04" builder_vsn: "5.2-3" otp_vsn: "25.3.2-2" @@ -115,7 +116,7 @@ jobs: echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT compile: - runs-on: ${{ fromJSON(needs.sanity-checks.outputs.runner_labels) }} + runs-on: ${{ fromJSON(needs.sanity-checks.outputs.xl_runner_labels) }} container: ${{ needs.sanity-checks.outputs.builder }} needs: - sanity-checks @@ -153,7 +154,7 @@ jobs: - compile uses: ./.github/workflows/run_emqx_app_tests.yaml with: - runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }} + runner_labels: ${{ needs.sanity-checks.outputs.xl_runner_labels }} builder: ${{ needs.sanity-checks.outputs.builder }} before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} after_ref: ${{ github.sha }} diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml index 5fc215f02..bb60eac16 100644 --- a/.github/workflows/run_helm_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -140,12 +140,13 @@ jobs: echo "waiting emqx started"; sleep 10; done - - name: Get Token + - name: Setup 18083 port forwarding + run: | + nohup kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null & + - name: Get auth token run: | - kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null & curl --head -X GET --retry 10 --retry-connrefused --retry-delay 6 http://localhost:18083/status echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV - - name: Check cluster timeout-minutes: 1 run: | diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 21f91624d..fe6ef5d43 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -71,7 +71,7 @@ jobs: ct_docker: runs-on: ${{ fromJSON(inputs.runner_labels) }} - name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})" + name: "${{ matrix.app }}-${{ matrix.suitegroup }} (${{ matrix.profile }})" strategy: fail-fast: false matrix: @@ -122,7 +122,7 @@ jobs: ct: runs-on: ${{ fromJSON(inputs.runner_labels) }} - name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})" + name: "${{ matrix.app }}-${{ matrix.suitegroup }} (${{ matrix.profile }})" strategy: fail-fast: false matrix: diff --git a/Makefile b/Makefile index 9d263a89e..654c372a6 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,11 @@ ifneq ($(CASES),) CASES_ARG := --case $(CASES) endif +# Allow user-set GROUPS environment variable +ifneq ($(GROUPS),) +GROUPS_ARG := --groups $(GROUPS) +endif + ## example: ## env SUITES=apps/appname/test/test_SUITE.erl CASES=t_foo make apps/appname-ct define gen-app-ct-target @@ -115,6 +120,7 @@ ifneq ($(SUITES),) --name $(CT_NODE_NAME) \ --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ --suite $(SUITES) \ + $(GROUPS_ARG) \ $(CASES_ARG) else @echo 'No suites found for $1' diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 09a91c9b1..13a84138d 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -109,22 +109,37 @@ fi ERLANG_CONTAINER='erlang' DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct" -case "${WHICH_APP}" in - lib-ee*) - ## ensure enterprise profile when testing lib-ee applications - export PROFILE='emqx-enterprise' - ;; - apps/*) - if [[ -f "${WHICH_APP}/BSL.txt" ]]; then - export PROFILE='emqx-enterprise' - else - export PROFILE='emqx' - fi - ;; - *) - export PROFILE="${PROFILE:-emqx}" - ;; -esac +if [ -z "${PROFILE+x}" ]; then + case "${WHICH_APP}" in + apps/emqx) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge) + export PROFILE='emqx-enterprise' + ;; + # emqx_connector test suite is using kafka bridge which is only available in emqx-enterprise + apps/emqx_connector) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_dashboard) + export PROFILE='emqx-enterprise' + ;; + lib-ee*) + ## ensure enterprise profile when testing lib-ee applications + export PROFILE='emqx-enterprise' + ;; + apps/*) + if [[ -f "${WHICH_APP}/BSL.txt" ]]; then + export PROFILE='emqx-enterprise' + else + export PROFILE='emqx' + fi + ;; + *) + export PROFILE="${PROFILE:-emqx}" + ;; + esac +fi if [ -f "$DOCKER_CT_ENVS_FILE" ]; then # shellcheck disable=SC2002 @@ -276,14 +291,18 @@ if [ "$STOP" = 'no' ]; then set -e fi -# rebar, mix and hex cache directory need to be writable by $DOCKER_USER -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache /.hex /.mix && chown $DOCKER_USER /.cache /.hex /.mix" -# need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 -hex 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie" -# the user must exist inside the container for `whoami` to work -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "useradd --uid $DOCKER_USER -M -d / emqx" || true -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $DOCKER_USER /var/lib/secret" || true -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "$INSTALL_ODBC" || true +if [ "$DOCKER_USER" != "root" ]; then + # the user must exist inside the container for `whoami` to work + docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c \ + "useradd --uid $DOCKER_USER -M -d / emqx && \ + mkdir -p /.cache /.hex /.mix && \ + chown $DOCKER_USER /.cache /.hex /.mix && \ + openssl rand -base64 -hex 16 > /.erlang.cookie && \ + chown $DOCKER_USER /.erlang.cookie && \ + chmod 0400 /.erlang.cookie && \ + chown -R $DOCKER_USER /var/lib/secret && \ + $INSTALL_ODBC" || true +fi if [ "$ONLY_UP" = 'yes' ]; then exit 0 diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 9120181c9..05c942421 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -54,7 +54,7 @@ fi ###### now deal with the github action's matrix. ################################################## -format_app_description() { +format_app_entry() { local groups="$2" local group=0 while [ "$groups" -gt $group ]; do @@ -72,48 +72,51 @@ END done } -describe_app() { - app="$1" - local runner="host" - local profile - if [ -f "${app}/docker-ct" ]; then - runner="docker" - fi - case "${app}" in - apps/*) - if [[ -f "${app}/BSL.txt" ]]; then - profile='emqx-enterprise' - else - profile='emqx' - fi - ;; - lib-ee/*) - profile='emqx-enterprise' - ;; - *) - echo "unknown app: $app" - exit 1 - ;; - esac - if [[ "$app" == "apps/emqx" ]]; then - suitegroups=5 - else - suitegroups=1 - fi - format_app_description "$app" "$suitegroups" "$profile" "$runner" -} - matrix() { - local sep='[' + local runner + local profile + local entries=() for app in ${APPS_ALL}; do - row="$(describe_app "$app")" - if [ -z "$row" ]; then - continue + if [ -f "${app}/docker-ct" ]; then + runner="docker" + else + runner="host" fi - echo -n "${sep}${row}" - sep=', ' + case "${app}" in + apps/emqx) + entries+=("$(format_app_entry "$app" 5 emqx "$runner")") + entries+=("$(format_app_entry "$app" 5 emqx-enterprise "$runner")") + ;; + apps/emqx_bridge) + entries+=("$(format_app_entry "$app" 1 emqx "$runner")") + entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") + ;; + apps/emqx_connector) + entries+=("$(format_app_entry "$app" 1 emqx "$runner")") + entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") + ;; + apps/emqx_dashboard) + entries+=("$(format_app_entry "$app" 1 emqx "$runner")") + entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") + ;; + apps/*) + if [[ -f "${app}/BSL.txt" ]]; then + profile='emqx-enterprise' + else + profile='emqx' + fi + entries+=("$(format_app_entry "$app" 1 "$profile" "$runner")") + ;; + lib-ee/*) + entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") + ;; + *) + echo "unknown app: $app" + exit 1 + ;; + esac done - echo ']' + echo -n "[$(IFS=,; echo "${entries[*]}")]" } matrix