yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci: make CI a little faster and less flaky 

- use latest ubuntu22.04 emqx-builder image in compose files
- use xl runners for compiling and for emqx app tests
- make helm tests less flaky
This commit is contained in:
Ivan Dyachkov 2023-10-30 10:32:27 +01:00
parent 45fdf8da1b
commit 045875d18d
8 changed files with 109 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/docker-compose-file/docker-compose-kafka.yaml
View File

@ -18,7 +18,7 @@ services:
- /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
kdc:
hostname: kdc.emqx.net
image: ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04
image: ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04
container_name: kdc.emqx.net
expose:
- 88 # kdc

16
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,17 +3,17 @@ version: '3.9'
services:
erlang:
container_name: erlang
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04}
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
env_file:
- conf.env
environment:
GITHUB_ACTIONS: ${GITHUB_ACTIONS}
GITHUB_TOKEN: ${GITHUB_TOKEN}
GITHUB_RUN_ID: ${GITHUB_RUN_ID}
GITHUB_SHA: ${GITHUB_SHA}
GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER}
GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}
GITHUB_REF: ${GITHUB_REF}
GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}
GITHUB_TOKEN: ${GITHUB_TOKEN:-}
GITHUB_RUN_ID: ${GITHUB_RUN_ID:-}
GITHUB_SHA: ${GITHUB_SHA:-}
GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER:-}
GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME:-}
GITHUB_REF: ${GITHUB_REF:-}
networks:
- emqx_bridge
ports:

5
.github/workflows/_pr_entrypoint.yaml vendored
View File

@ -25,6 +25,7 @@ jobs:
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted","ephemeral","linux","x64"]' || '["ubuntu-22.04"]' }}
xl_runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted","ephemeral-xl","linux","x64"]' || '["ubuntu-22.04"]' }}
builder: "ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04"
builder_vsn: "5.2-3"
otp_vsn: "25.3.2-2"
@ -115,7 +116,7 @@ jobs:
echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
compile:
runs-on: ${{ fromJSON(needs.sanity-checks.outputs.runner_labels) }}
runs-on: ${{ fromJSON(needs.sanity-checks.outputs.xl_runner_labels) }}
container: ${{ needs.sanity-checks.outputs.builder }}
needs:
- sanity-checks
@ -153,7 +154,7 @@ jobs:
- compile
uses: ./.github/workflows/run_emqx_app_tests.yaml
with:
runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
runner_labels: ${{ needs.sanity-checks.outputs.xl_runner_labels }}
builder: ${{ needs.sanity-checks.outputs.builder }}
before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
after_ref: ${{ github.sha }}

7
.github/workflows/run_helm_tests.yaml vendored
View File

@ -140,12 +140,13 @@ jobs:
echo "waiting emqx started";
sleep 10;
done
- name: Get Token
- name: Setup 18083 port forwarding
run: |
nohup kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null &
- name: Get auth token
run: |
kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null &
curl --head -X GET --retry 10 --retry-connrefused --retry-delay 6 http://localhost:18083/status
echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV
- name: Check cluster
timeout-minutes: 1
run: |

4
.github/workflows/run_test_cases.yaml vendored
View File

@ -71,7 +71,7 @@ jobs:
ct_docker:
runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})"
name: "${{ matrix.app }}-${{ matrix.suitegroup }} (${{ matrix.profile }})"
strategy:
fail-fast: false
matrix:
@ -122,7 +122,7 @@ jobs:
ct:
runs-on: ${{ fromJSON(inputs.runner_labels) }}
name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})"
name: "${{ matrix.app }}-${{ matrix.suitegroup }} (${{ matrix.profile }})"
strategy:
fail-fast: false
matrix:

6
Makefile
View File

@ -104,6 +104,11 @@ ifneq ($(CASES),)
CASES_ARG := --case $(CASES)
endif
# Allow user-set GROUPS environment variable
ifneq ($(GROUPS),)
GROUPS_ARG := --groups $(GROUPS)
endif
## example:
## env SUITES=apps/appname/test/test_SUITE.erl CASES=t_foo make apps/appname-ct
define gen-app-ct-target
@ -115,6 +120,7 @@ ifneq ($(SUITES),)
--name $(CT_NODE_NAME) \
--cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \
--suite $(SUITES) \
$(GROUPS_ARG) \
$(CASES_ARG)
else
@echo 'No suites found for $1'

39
scripts/ct/run.sh
View File

@ -109,7 +109,21 @@ fi
ERLANG_CONTAINER='erlang'
DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct"
case "${WHICH_APP}" in
if [ -z "${PROFILE+x}" ]; then
case "${WHICH_APP}" in
apps/emqx)
export PROFILE='emqx-enterprise'
;;
apps/emqx_bridge)
export PROFILE='emqx-enterprise'
;;
# emqx_connector test suite is using kafka bridge which is only available in emqx-enterprise
apps/emqx_connector)
export PROFILE='emqx-enterprise'
;;
apps/emqx_dashboard)
export PROFILE='emqx-enterprise'
;;
lib-ee*)
## ensure enterprise profile when testing lib-ee applications
export PROFILE='emqx-enterprise'
@ -124,7 +138,8 @@ case "${WHICH_APP}" in
*)
export PROFILE="${PROFILE:-emqx}"
;;
esac
esac
fi
if [ -f "$DOCKER_CT_ENVS_FILE" ]; then
# shellcheck disable=SC2002
@ -276,14 +291,18 @@ if [ "$STOP" = 'no' ]; then
set -e
fi
# rebar, mix and hex cache directory need to be writable by $DOCKER_USER
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache /.hex /.mix && chown $DOCKER_USER /.cache /.hex /.mix"
# need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 -hex 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie"
# the user must exist inside the container for `whoami` to work
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "useradd --uid $DOCKER_USER -M -d / emqx" || true
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $DOCKER_USER /var/lib/secret" || true
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "$INSTALL_ODBC" || true
if [ "$DOCKER_USER" != "root" ]; then
# the user must exist inside the container for `whoami` to work
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c \
"useradd --uid $DOCKER_USER -M -d / emqx && \
mkdir -p /.cache /.hex /.mix && \
chown $DOCKER_USER /.cache /.hex /.mix && \
openssl rand -base64 -hex 16 > /.erlang.cookie && \
chown $DOCKER_USER /.erlang.cookie && \
chmod 0400 /.erlang.cookie && \
chown -R $DOCKER_USER /var/lib/secret && \
$INSTALL_ODBC" || true
fi
if [ "$ONLY_UP" = 'yes' ]; then
exit 0

49
scripts/find-apps.sh
View File

@ -54,7 +54,7 @@ fi
###### now deal with the github action's matrix.
##################################################
format_app_description() {
format_app_entry() {
local groups="$2"
local group=0
while [ "$groups" -gt $group ]; do
@ -72,48 +72,51 @@ END
done
}
describe_app() {
app="$1"
local runner="host"
matrix() {
local runner
local profile
local entries=()
for app in ${APPS_ALL}; do
if [ -f "${app}/docker-ct" ]; then
runner="docker"
else
runner="host"
fi
case "${app}" in
apps/emqx)
entries+=("$(format_app_entry "$app" 5 emqx "$runner")")
entries+=("$(format_app_entry "$app" 5 emqx-enterprise "$runner")")
;;
apps/emqx_bridge)
entries+=("$(format_app_entry "$app" 1 emqx "$runner")")
entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
;;
apps/emqx_connector)
entries+=("$(format_app_entry "$app" 1 emqx "$runner")")
entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
;;
apps/emqx_dashboard)
entries+=("$(format_app_entry "$app" 1 emqx "$runner")")
entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
;;
apps/*)
if [[ -f "${app}/BSL.txt" ]]; then
profile='emqx-enterprise'
else
profile='emqx'
fi
entries+=("$(format_app_entry "$app" 1 "$profile" "$runner")")
;;
lib-ee/*)
profile='emqx-enterprise'
entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
;;
*)
echo "unknown app: $app"
exit 1
;;
esac
if [[ "$app" == "apps/emqx" ]]; then
suitegroups=5
else
suitegroups=1
fi
format_app_description "$app" "$suitegroups" "$profile" "$runner"
}
matrix() {
local sep='['
for app in ${APPS_ALL}; do
row="$(describe_app "$app")"
if [ -z "$row" ]; then
continue
fi
echo -n "${sep}${row}"
sep=', '
done
echo ']'
echo -n "[$(IFS=,; echo "${entries[*]}")]"
}
matrix