ci: refactor workflows to use single entry point on pull request trigger

This commit is contained in:
Ivan Dyachkov 2023-07-12 15:55:38 +02:00
parent 665695a977
commit 362235021c
30 changed files with 1158 additions and 1223 deletions

View File

@ -19,7 +19,9 @@ services:
- emqx2 - emqx2
volumes: volumes:
- ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
- ../../apps/emqx/etc/certs:/usr/local/etc/haproxy/certs - ../../apps/emqx/etc/certs/cert.pem:/usr/local/etc/haproxy/certs/cert.pem
- ../../apps/emqx/etc/certs/key.pem:/usr/local/etc/haproxy/certs/key.pem
- ../../apps/emqx/etc/certs/cacert.pem:/usr/local/etc/haproxy/certs/cacert.pem
ports: ports:
- "18083:18083" - "18083:18083"
# - "1883:1883" # - "1883:1883"
@ -34,7 +36,7 @@ services:
- -c - -c
- | - |
set -x set -x
cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /tmp/emqx.pem cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /var/lib/haproxy/emqx.pem
haproxy -f /usr/local/etc/haproxy/haproxy.cfg haproxy -f /usr/local/etc/haproxy/haproxy.cfg
emqx1: emqx1:

View File

@ -83,13 +83,13 @@ backend emqx_ws_back
frontend emqx_ssl frontend emqx_ssl
mode tcp mode tcp
option tcplog option tcplog
bind *:8883 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 bind *:8883 ssl crt /var/lib/haproxy/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3
default_backend emqx_ssl_back default_backend emqx_ssl_back
frontend emqx_wss frontend emqx_wss
mode tcp mode tcp
option tcplog option tcplog
bind *:8084 ssl crt /tmp/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 bind *:8084 ssl crt /var/lib/haproxy/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3
default_backend emqx_wss_back default_backend emqx_wss_back
backend emqx_ssl_back backend emqx_ssl_back

32
.ci/pr-sanity-checks.sh Executable file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
if ! type "docker" > /dev/null; then
echo "docker is not installed"
exit 1
fi
if ! type "yq" > /dev/null; then
echo "yq is not installed"
exit 1
fi
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.1-1}
EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-25.3.2-1}
EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.14.5}
EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04}
EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}}
commands=$(yq ".runs.steps[].run" .github/actions/pr-sanity-checks/action.yaml | grep -v null)
BEFORE_REF=${BEFORE_REF:-$(git rev-parse master)}
AFTER_REF=${AFTER_REF:-$(git rev-parse HEAD)}
docker run --rm -it -v "$(pwd):/emqx" -w /emqx \
-e GITHUB_WORKSPACE=/emqx \
-e BEFORE_REF="$BEFORE_REF" \
-e AFTER_REF="$AFTER_REF" \
-e GITHUB_BASE_REF="$BEFORE_REF" \
-e MIX_ENV=emqx-enterprise \
-e PROFILE=emqx-enterprise \
"${EMQX_BUILDER}" /bin/bash -c "${commands}"

View File

@ -0,0 +1,68 @@
name: 'Pull request sanity checks'
defaults:
run:
shell: 'bash -Eeuo pipefail {0}'
runs:
using: composite
steps:
- name: Work around https://github.com/actions/checkout/issues/766
shell: bash
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Run gitlint
shell: bash
env:
BEFORE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
AFTER_REF: ${{ github.sha }}
run: |
pip install gitlint
gitlint --commits $BEFORE_REF..$AFTER_REF --config .github/workflows/.gitlint
- name: Run shellcheck
shell: bash
run: |
apt-get update -y && apt-get install -y shellcheck
./scripts/shellcheck.sh
# https://github.com/rhysd/actionlint/blob/main/docs/usage.md#use-actionlint-on-github-actions
- name: Check workflow files
shell: bash
run: |
bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
# TODO: enable shellcheck when all the current issues are fixed
./actionlint -color \
-shellcheck= \
-ignore 'label ".+" is unknown' \
-ignore 'value "emqx-enterprise" in "exclude"'
- name: Check line-break at EOF
shell: bash
run: |
./scripts/check-nl-at-eof.sh
- name: Check apps version
shell: bash
run: |
./scripts/apps-version-check.sh
- name: Check Erlang code formatting
shell: bash
run: |
echo "==> ./scripts/check-format.sh"
./scripts/check-format.sh
echo "==> check-format ok"
- name: Run elvis check
shell: bash
run: |
./scripts/elvis-check.sh $GITHUB_BASE_REF
- name: Setup mix
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
shell: bash
run: |
mix local.hex --force && mix local.rebar --force
- name: Check Elixir code formatting
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
shell: bash
run: |
mix format --check-formatted

View File

@ -0,0 +1,49 @@
name: 'Run jmeter'
inputs:
version-emqx:
required: true
type: string
runs:
using: composite
steps:
- uses: actions/download-artifact@v3
with:
name: emqx-docker
path: /tmp
- name: load docker image
shell: bash
env:
PKG_VSN: ${{ inputs.version-emqx }}
run: |
EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
ref: broker-autotest-v5
path: scripts
- uses: actions/setup-java@v3
with:
java-version: '8.0.282' # The JDK version to make available on the path.
java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@v3
with:
name: apache-jmeter.tgz
- name: install jmeter
shell: bash
env:
JMETER_VERSION: 5.4.3
run: |
tar -xf apache-jmeter.tgz
ln -s apache-jmeter-$JMETER_VERSION jmeter
echo "jmeter.save.saveservice.output_format=xml" >> jmeter/user.properties
echo "jmeter.save.saveservice.response_data.on_error=true" >> jmeter/user.properties
cd jmeter/lib/ext
wget --no-verbose https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
wget --no-verbose https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
wget --no-verbose https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar

194
.github/workflows/_pr_entrypoint.yaml vendored Normal file
View File

@ -0,0 +1,194 @@
name: PR Entrypoint
on:
pull_request:
env:
IS_CI: "yes"
OTP_VSN: "25.3.2-1"
ELIXIR_VSN: "1.14.5"
BUILDER_VSN: "25.3.2-1"
jobs:
sanity-checks:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
builder: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
builder_vsn: "5.1-3"
otp_vsn: "25.3.2-1"
elixir_vsn: "1.14.5"
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: ./.github/actions/pr-sanity-checks
- name: Build matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.1-3",
otp: "25.3.2-1",
elixir: "1.14.5"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.1-3",
otp: ["25.3.2-1"][],
elixir: "1.14.5"
})
]
')"
echo "${MATRIX}" | jq
CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT
echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT
echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT
echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
compile:
runs-on: ${{ needs.sanity-checks.outputs.runner }}
container: ${{ needs.sanity-checks.outputs.builder }}
needs:
- sanity-checks
strategy:
matrix:
profile:
- emqx
- emqx-enterprise
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- id: compile
env:
PROFILE: ${{ matrix.profile }}
ENABLE_COVER_COMPILE: 1
run: |
make $PROFILE
zip -ryq $PROFILE.zip .
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.profile }}
path: ${{ matrix.profile }}.zip
retention-days: 1
run_test_cases:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/run_test_cases.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
builder: ${{ needs.sanity-checks.outputs.builder }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
static_checks:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/static_checks.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
builder: ${{ needs.sanity-checks.outputs.builder }}
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
build_slim_packages:
needs:
- sanity-checks
uses: ./.github/workflows/build_slim_packages.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
builder: ${{ needs.sanity-checks.outputs.builder }}
builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }}
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
build_docker_for_test:
needs:
- sanity-checks
uses: ./.github/workflows/build_docker_for_test.yaml
with:
otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
spellcheck:
needs:
- sanity-checks
- build_slim_packages
uses: ./.github/workflows/spellcheck.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
run_conf_tests:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/run_conf_tests.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
builder: ${{ needs.sanity-checks.outputs.builder }}
check_deps_integrity:
needs:
- sanity-checks
uses: ./.github/workflows/check_deps_integrity.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
builder: ${{ needs.sanity-checks.outputs.builder }}
elixir_release:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/elixir_release.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
builder: ${{ needs.sanity-checks.outputs.builder }}
run_jmeter_tests:
needs:
- sanity-checks
- build_docker_for_test
uses: ./.github/workflows/run_jmeter_tests.yaml
with:
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
run_docker_tests:
needs:
- sanity-checks
- build_docker_for_test
uses: ./.github/workflows/run_docker_tests.yaml
with:
runner: ${{ needs.sanity-checks.outputs.runner }}
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
run_helm_tests:
needs:
- sanity-checks
- build_docker_for_test
uses: ./.github/workflows/run_helm_tests.yaml
with:
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}

View File

@ -1,14 +0,0 @@
name: Check Apps Version
on: [pull_request]
jobs:
check_apps_version:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Check apps version
run: ./scripts/apps-version-check.sh

View File

@ -5,11 +5,13 @@ concurrency:
cancel-in-progress: true cancel-in-progress: true
on: on:
push: workflow_call:
tags: # on:
- v* # push:
- e* # tags:
- docker-latest-* # - v*
# - e*
# - docker-latest-*
workflow_dispatch: workflow_dispatch:
inputs: inputs:
branch_or_tag: branch_or_tag:

View File

@ -0,0 +1,78 @@
name: Build docker image for test
concurrency:
group: docker-test-build-${{ github.event_name }}-${{ github.sha }}
cancel-in-progress: true
on:
workflow_call:
inputs:
otp_vsn:
required: true
type: string
elixir_vsn:
required: true
type: string
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
# on:
# push:
# branches:
# - master
# - release-51
# pull_request:
# # GitHub pull_request action is by default triggered when
# # opened reopened or synchronize,
# # we add labeled and unlabeled to the list because
# # the mac job dpends on the PR having a 'Mac' label
# types:
# - labeled
# - unlabeled
# - opened
# - reopened
# - synchronize
# workflow_dispatch:
jobs:
docker:
runs-on: ubuntu-latest
env:
EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
OTP_VSN: ${{ inputs.otp_vsn }}
ELIXIR_VSN: ${{ inputs.elixir_vsn }}
strategy:
fail-fast: false
matrix:
profile:
- emqx
- emqx-enterprise
- emqx-elixir
steps:
- uses: actions/checkout@v3
- name: build and export to Docker
id: build
run: |
make ${EMQX_NAME}-docker
echo "EMQX_IMAGE_TAG=$(cat .docker_image_tag)" >> $GITHUB_ENV
- name: smoke test
run: |
CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
docker stop $CID
- name: export docker image
run: |
docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
- uses: actions/upload-artifact@v3
with:
name: "${{ env.EMQX_NAME }}-docker"
path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"
retention-days: 3

View File

@ -5,12 +5,14 @@ concurrency:
cancel-in-progress: true cancel-in-progress: true
on: on:
push: workflow_call:
branches: # on:
- 'ci/**' # push:
tags: # branches:
- v* # - 'ci/**'
- e* # tags:
# - v*
# - e*
workflow_dispatch: workflow_dispatch:
inputs: inputs:
branch_or_tag: branch_or_tag:
@ -88,7 +90,7 @@ jobs:
fetch-depth: 0 fetch-depth: 0
- uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: ilammy/msvc-dev-cmd@v1.12.0
- uses: erlef/setup-beam@v1.15.4 - uses: erlef/setup-beam@v1.16.0
with: with:
otp-version: 25.3.2 otp-version: 25.3.2
- name: build - name: build

View File

@ -1,58 +1,69 @@
name: Build slim packages name: Build slim packages
concurrency: concurrency:
group: slim-${{ github.event_name }}-${{ github.ref }} group: slim-${{ github.event_name }}-${{ github.sha }}
cancel-in-progress: true cancel-in-progress: true
on: on:
push: workflow_call:
branches: inputs:
- master runner:
- release-51 required: true
pull_request: type: string
# GitHub pull_request action is by default triggered when builder:
# opened reopened or synchronize, required: true
# we add labeled and unlabeled to the list because type: string
# the mac job dpends on the PR having a 'Mac' label builder_vsn:
types: required: true
- labeled type: string
- unlabeled otp_vsn:
- opened required: true
- reopened type: string
- synchronize elixir_vsn:
required: true
type: string
# on:
# push:
# branches:
# - master
# - release-51
# pull_request:
# # GitHub pull_request action is by default triggered when
# # opened reopened or synchronize,
# # we add labeled and unlabeled to the list because
# # the mac job dpends on the PR having a 'Mac' label
# types:
# - labeled
# - unlabeled
# - opened
# - reopened
# - synchronize
workflow_dispatch: workflow_dispatch:
jobs: jobs:
linux: linux:
runs-on: aws-amd64 runs-on: ${{ inputs.runner }}
env:
EMQX_NAME: ${{ matrix.profile[0] }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: profile:
- ["emqx", "25.3.2-1", "el7", "erlang"] - ["emqx", "25.3.2-1", "ubuntu20.04", "elixir"]
- ["emqx", "25.3.2-1", "ubuntu22.04", "elixir"]
- ["emqx-enterprise", "25.3.2-1", "amzn2023", "erlang"]
- ["emqx-enterprise", "25.3.2-1", "ubuntu20.04", "erlang"] - ["emqx-enterprise", "25.3.2-1", "ubuntu20.04", "erlang"]
builder:
- 5.1-3
elixir:
- '1.14.5'
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0
- name: prepare
run: |
echo "EMQX_NAME=${{ matrix.profile[0] }}" >> $GITHUB_ENV
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
- name: Work around https://github.com/actions/checkout/issues/766 - name: Work around https://github.com/actions/checkout/issues/766
run: | run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE" git config --global --add safe.directory "$GITHUB_WORKSPACE"
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
- name: build and test tgz package - name: build and test tgz package
if: matrix.profile[3] == 'erlang' if: matrix.profile[3] == 'erlang'
run: | run: |
@ -77,12 +88,14 @@ jobs:
with: with:
name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
path: _packages/${{ matrix.profile[0] }}/* path: _packages/${{ matrix.profile[0] }}/*
retention-days: 7
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: "${{ matrix.profile[0] }}_schema_dump" name: "${{ matrix.profile[0] }}_schema_dump"
path: | path: |
scripts/spellcheck scripts/spellcheck
_build/docgen/${{ matrix.profile[0] }}/schema-en.json _build/docgen/${{ matrix.profile[0] }}/schema-en.json
retention-days: 7
windows: windows:
runs-on: windows-2019 runs-on: windows-2019
@ -96,7 +109,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: ilammy/msvc-dev-cmd@v1.12.0
- uses: erlef/setup-beam@v1.15.4 - uses: erlef/setup-beam@v1.16.0
with: with:
otp-version: ${{ matrix.otp }} otp-version: ${{ matrix.otp }}
- name: build - name: build
@ -129,6 +142,7 @@ jobs:
with: with:
name: windows name: windows
path: _packages/${{ matrix.profile }}/* path: _packages/${{ matrix.profile }}/*
retention-days: 7
mac: mac:
strategy: strategy:
@ -136,20 +150,18 @@ jobs:
matrix: matrix:
profile: profile:
- emqx - emqx
- emqx-enterprise
otp: otp:
- 25.3.2-1 - ${{ inputs.otp_vsn }}
os: os:
- macos-11 - macos-11
- macos-12-arm64 - macos-12-arm64
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
env:
EMQX_NAME: ${{ matrix.profile }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: prepare
run: |
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
- uses: ./.github/actions/package-macos - uses: ./.github/actions/package-macos
with: with:
profile: ${{ matrix.profile }} profile: ${{ matrix.profile }}
@ -163,84 +175,4 @@ jobs:
with: with:
name: ${{ matrix.os }} name: ${{ matrix.os }}
path: _packages/**/* path: _packages/**/*
retention-days: 7
docker:
runs-on: aws-amd64
strategy:
fail-fast: false
matrix:
profile:
- ["emqx", "5.0.16"]
- ["emqx-enterprise", "5.0.1"]
steps:
- uses: actions/checkout@v3
- name: prepare
run: |
EMQX_NAME=${{ matrix.profile[0] }}
PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}
EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test
EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }}
echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV
echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV
echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV
- uses: docker/setup-buildx-action@v2
- name: build and export to Docker
uses: docker/build-push-action@v4
with:
context: .
file: ./deploy/docker/Dockerfile
load: true
tags: ${{ env.EMQX_IMAGE_TAG }}
build-args: |
EMQX_NAME=${{ env.EMQX_NAME }}
- name: smoke test
run: |
CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
docker stop $CID
- name: dashboard tests
working-directory: ./scripts/ui-tests
run: |
set -eu
docker compose up --abort-on-container-exit --exit-code-from selenium
- name: test two nodes cluster with proto_dist=inet_tls in docker
run: |
./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
# cleanup
./scripts/test/start-two-nodes-in-docker.sh -c
- name: export docker image
run: |
docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz
- uses: actions/upload-artifact@v3
with:
name: "${{ matrix.profile[0] }}-docker"
path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz"
- name: cleanup
if: always()
working-directory: ./scripts/ui-tests
run: |
docker compose rm -fs
spellcheck:
needs: linux
strategy:
matrix:
profile:
- emqx
- emqx-enterprise
runs-on: aws-amd64
steps:
- uses: actions/download-artifact@v3
name: Download schema dump
with:
name: "${{ matrix.profile }}_schema_dump"
path: /tmp/
- name: Run spellcheck
run: |
bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json

View File

@ -1,14 +1,46 @@
name: Check Rebar Dependencies name: Check integrity of rebar and mix dependencies
on: on:
pull_request: workflow_call:
inputs:
runner:
required: true
type: string
builder:
required: true
type: string
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ubuntu-22.04 runs-on: ${{ inputs.runner }}
container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04 container: ${{ inputs.builder }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Run check-deps-integrity.escript - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
run: ./scripts/check-deps-integrity.escript - run: make ensure-rebar3
- run: ./scripts/check-deps-integrity.escript
- name: Setup mix
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
run: |
mix local.hex --force
mix local.rebar --force
mix deps.get
- run: ./scripts/check-elixir-deps-discrepancies.exs
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- run: ./scripts/check-elixir-applications.exs
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- name: Upload produced lock files
uses: actions/upload-artifact@v3
if: failure()
with:
name: produced_lock_files
path: |
mix.lock
rebar.lock
retention-days: 1

View File

@ -1,29 +0,0 @@
name: Code style check
on: [pull_request]
jobs:
code_style_check:
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1000
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Check line-break at EOF
run: |
./scripts/check-nl-at-eof.sh
- name: Check Elixir code formatting
run: |
mix format --check-formatted
- name: Check Erlang code formatting
run: |
./scripts/check-format.sh
- name: Run elvis check
run: |
./scripts/elvis-check.sh $GITHUB_BASE_REF

View File

@ -1,45 +0,0 @@
---
name: Check Elixir Release Applications
on:
pull_request:
jobs:
elixir_apps_check:
runs-on: ubuntu-22.04
# just use the latest builder
container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
strategy:
fail-fast: false
matrix:
profile:
- emqx
- emqx-enterprise
- emqx-pkg
- emqx-enterprise-pkg
steps:
- name: fix_git_permission
run: git config --global --add safe.directory '/__w/emqx/emqx'
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: ensure rebar
run: ./scripts/ensure-rebar3.sh
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: check applications
run: ./scripts/check-elixir-applications.exs
env:
MIX_ENV: ${{ matrix.profile }}
PROFILE: ${{ matrix.profile }}
# - name: check applications started with emqx_machine
# run: ./scripts/check-elixir-emqx-machine-boot-discrepancies.exs
# env:
# MIX_ENV: ${{ matrix.profile }}
...

View File

@ -1,49 +0,0 @@
---
name: Elixir Dependency Version Check
on:
pull_request:
jobs:
elixir_deps_check:
runs-on: ubuntu-22.04
container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04
steps:
- name: Checkout
uses: actions/checkout@v3
- name: ensure rebar
run: ./scripts/ensure-rebar3.sh
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: setup mix
run: |
mix local.hex --force
mix local.rebar --force
mix deps.get
# we check only enterprise because `rebar3 tree`, even if an
# enterprise app is excluded from `project_app_dirs` in
# `rebar.config.erl`, will still list dependencies from it.
# Since the enterprise profile is a superset of the
# community one and thus more complete, we use the former.
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- name: check elixir deps
run: ./scripts/check-elixir-deps-discrepancies.exs
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- name: produced lock files
uses: actions/upload-artifact@v3
if: failure()
with:
name: produced_lock_files
path: |
mix.lock
rebar.lock
retention-days: 1
...

47
.github/workflows/elixir_release.yaml vendored Normal file
View File

@ -0,0 +1,47 @@
# FIXME: temporary workflow for testing; remove later
name: Elixir Build (temporary)
concurrency:
group: elixir-release-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_call:
inputs:
runner:
required: true
type: string
builder:
required: true
type: string
jobs:
elixir_release_build:
runs-on: ${{ inputs.runner }}
container: ${{ inputs.builder }}
strategy:
matrix:
profile:
- emqx
steps:
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: elixir release
run: make ${{ matrix.profile }}-elixir
- name: start release
run: |
_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
- name: check if started
run: |
set -e
sleep 10
timeout 1 bash -c '</dev/tcp/localhost/1883'
_build/${{ matrix.profile }}/rel/emqx/bin/emqx ping
_build/${{ matrix.profile }}/rel/emqx/bin/emqx ctl status
./scripts/test/emqx-smoke-test.sh localhost 18083

View File

@ -1,41 +0,0 @@
# FIXME: temporary workflow for testing; remove later
name: Elixir Build (temporary)
concurrency:
group: mix-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
workflow_dispatch:
jobs:
elixir_release_build:
runs-on: ubuntu-22.04
strategy:
matrix:
profile:
- emqx
- emqx-enterprise
container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04
steps:
- name: Checkout
uses: actions/checkout@v3
- name: install tools
run: apt update && apt install netcat-openbsd
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: elixir release
run: make ${{ matrix.profile }}-elixir
- name: start release
run: |
cd _build/${{ matrix.profile }}/rel/emqx
bin/emqx start
- name: check if started
run: |
sleep 10
nc -zv localhost 1883
cd _build/${{ matrix.profile }}/rel/emqx
bin/emqx ping
bin/emqx ctl status

View File

@ -1,53 +1,59 @@
name: Run Configuration tests name: Run Configuration tests
concurrency: concurrency:
group: conftest-${{ github.event_name }}-${{ github.ref }} group: conftest-${{ github.event_name }}-${{ github.sha }}
cancel-in-progress: true cancel-in-progress: true
on: on:
push: workflow_call:
branches: inputs:
- master runner:
- 'ci/**' required: true
tags: type: string
- v* builder:
- e* required: true
pull_request: type: string
env: # on:
IS_CI: "yes" # push:
# branches:
# - master
# - 'ci/**'
# tags:
# - v*
# - e*
# pull_request:
jobs: jobs:
run_conf_tests: run_conf_tests:
runs-on: ubuntu-22.04 runs-on: ${{ inputs.runner }}
container: ${{ inputs.builder }}
env:
PROFILE: ${{ matrix.profile }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: profile:
- emqx - emqx
- emqx-enterprise - emqx-enterprise
container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3 - uses: actions/download-artifact@v3
with: with:
path: source name: ${{ matrix.profile }}
- name: build_package - name: extract artifact
working-directory: source
run: | run: |
make ${{ matrix.profile }} unzip -o -q ${{ matrix.profile }}.zip
- name: run_tests git config --global --add safe.directory "$GITHUB_WORKSPACE"
working-directory: source - run: ./scripts/test/check-example-configs.sh
env: - run: ./scripts/conf-test/run.sh
PROFILE: ${{ matrix.profile }} - name: print erlang log
run: |
./scripts/conf-test/run.sh
- name: print_erlang_log
if: failure() if: failure()
run: | run: |
cat source/_build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.* cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.profile }} name: logs-${{ matrix.profile }}
path: source/_build/${{ matrix.profile }}/rel/emqx/logs path: _build/${{ matrix.profile }}/rel/emqx/logs

127
.github/workflows/run_docker_tests.yaml vendored Normal file
View File

@ -0,0 +1,127 @@
name: Docker image tests
concurrency:
group: docker-tests-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_call:
inputs:
runner:
required: true
type: string
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
# on:
# push:
# branches:
# - master
# - 'ci/**'
# tags:
# - v*
# pull_request:
jobs:
basic-tests:
runs-on: ${{ inputs.runner }}
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix:
profile:
- ["emqx", "emqx/emqx:5.0.16"]
- ["emqx-enterprise", "emqx/emqx-enterprise:5.0.1"]
env:
EMQX_NAME: ${{ matrix.profile[0] }}
PKG_VSN: ${{ matrix.profile[0] == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
EMQX_IMAGE_OLD_VERSION_TAG: ${{ matrix.profile[1] }}
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: ${{ env.EMQX_NAME }}-docker
path: /tmp
- name: load docker image
run: |
EMQX_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g')
echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
- name: dashboard tests
working-directory: ./scripts/ui-tests
run: |
set -eu
docker compose up --abort-on-container-exit --exit-code-from selenium
- name: test two nodes cluster with proto_dist=inet_tls in docker
run: |
./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
./scripts/test/start-two-nodes-in-docker.sh -c
- name: cleanup
if: always()
working-directory: ./scripts/ui-tests
run: |
docker compose rm -fs
paho-mqtt-testing:
runs-on: ${{ inputs.runner }}
defaults:
run:
shell: bash
env:
EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
_EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }}
strategy:
fail-fast: false
matrix:
profile:
- emqx
- emqx-enterprise
- emqx-elixir
cluster_db_backend:
- mnesia
- rlog
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: ${{ env.EMQX_NAME }}-docker
path: /tmp
- name: load docker image
run: |
EMQX_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
- name: run emqx
timeout-minutes: 5
run: |
./.ci/docker-compose-file/scripts/run-emqx.sh $_EMQX_DOCKER_IMAGE_TAG $_EMQX_TEST_DB_BACKEND
- name: make paho tests
run: |
if ! docker exec -i python /scripts/pytest.sh "$_EMQX_TEST_DB_BACKEND"; then
echo "DUMP_CONTAINER_LOGS_BGN"
echo "============== haproxy =============="
docker logs haproxy
echo "============== node1 =============="
docker logs node1.emqx.io
echo "============== node2 =============="
docker logs node2.emqx.io
echo "DUMP_CONTAINER_LOGS_END"
exit 1
fi
# node_dump requires netstat, which is not available in the container
# simple smoke test for node_dump
# - name: test node_dump
# run: |
# docker exec -u root node1.emqx.io apt update && apt install -y net-tools
# docker exec node1.emqx.io node_dump

View File

@ -1,80 +0,0 @@
name: Check emqx app standalone
## apps/emqx can be used as a rebar/mix dependency
## in other project, so we need to make sure apps/emqx
## as an Erlang/Elixir app works standalone
on:
pull_request:
jobs:
run_emqx_app_tests:
strategy:
matrix:
builder:
- 5.1-3
otp:
- 25.3.2-1
# no need to use more than 1 version of Elixir, since tests
# run using only Erlang code. This is needed just to specify
# the base image.
elixir:
- 1.14.5
os:
- ubuntu22.04
arch:
- amd64
runs-on:
- aws-amd64
- ubuntu-22.04
use-self-hosted:
- ${{ github.repository_owner == 'emqx' }}
exclude:
- runs-on: ubuntu-22.04
use-self-hosted: true
- runs-on: aws-amd64
use-self-hosted: false
runs-on: ${{ matrix.runs-on }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: run
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
echo "git diff base: $GITHUB_BASE_REF"
if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then
# base is a commit sha1
compare_base="$GITHUB_BASE_REF"
else
repo="${GITHUB_REPOSITORY}"
git remote -v
remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')"
git fetch "$remote" "$GITHUB_BASE_REF"
compare_base="$remote/$GITHUB_BASE_REF"
fi
changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)"
if [ "$changed_files" = '' ]; then
echo "nothing changed in apps/emqx, ignored."
exit 0
fi
make ensure-rebar3
cp rebar3 apps/emqx/
cd apps/emqx
./rebar3 xref
./rebar3 dialyzer
./rebar3 eunit -v
./rebar3 ct --name 'test@127.0.0.1' -v --readable=true
./rebar3 proper -d test/props
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs-${{ matrix.runs-on }}
path: apps/emqx/_build/test/logs

View File

@ -1,255 +0,0 @@
name: Functional Verification Tests
concurrency:
group: fvt-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches:
- master
- 'ci/**'
tags:
- v*
pull_request:
jobs:
prepare:
runs-on: ubuntu-22.04
# prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-debian11
steps:
- uses: actions/checkout@v3
with:
path: source
fetch-depth: 0
- name: get deps
run: |
make -C source deps-all
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source
path: source.zip
docker_test:
runs-on: ubuntu-22.04
needs: prepare
strategy:
fail-fast: false
matrix:
profile:
- emqx
- emqx-enterprise
- emqx-elixir
cluster_db_backend:
- mnesia
- rlog
os:
- ["debian11", "debian:11-slim"]
builder:
- 5.1-3
otp:
- 25.3.2-1
elixir:
- 1.14.5
arch:
- amd64
steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: unzip -q source.zip
- name: make docker image
working-directory: source
env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
EMQX_RUNNER: ${{ matrix.os[1] }}
run: |
make ${{ matrix.profile }}-docker
- name: run emqx
timeout-minutes: 5
working-directory: source
run: |
set -x
if [[ "${{ matrix.profile }}" = *-elixir ]]
then
export IS_ELIXIR=yes
PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g")
IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir
else
IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
fi
./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
- name: make paho tests
run: |
if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then
echo "DUMP_CONTAINER_LOGS_BGN"
echo "============== haproxy =============="
docker logs haproxy
echo "============== node1 =============="
docker logs node1.emqx.io
echo "============== node2 =============="
docker logs node2.emqx.io
echo "DUMP_CONTAINER_LOGS_END"
exit 1
fi
# simple smoke test for node_dump
- name: test node_dump
run: |
docker exec node1.emqx.io node_dump
helm_test:
runs-on: ubuntu-22.04
needs: prepare
strategy:
fail-fast: false
matrix:
discovery:
- k8s
- dns
profile:
- emqx
- emqx-enterprise
os:
- ["debian11", "debian:11-slim"]
builder:
- 5.1-3
otp:
- 25.3.2-1
elixir:
- 1.14.5
arch:
- amd64
# - emqx-enterprise # TODO test enterprise
steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: unzip -q source.zip
- name: make docker image
working-directory: source
env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
EMQX_RUNNER: ${{ matrix.os[1] }}
run: |
make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
- run: minikube start
- run: minikube image load $TARGET:$EMQX_TAG
- name: run emqx on chart
working-directory: source
if: matrix.discovery == 'k8s'
run: |
helm install ${{ matrix.profile }} \
--set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
--set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
--set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${{ matrix.profile }}-headless" \
--set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \
--set image.repository=$TARGET \
--set image.pullPolicy=Never \
--set image.tag=$EMQX_TAG \
--set emqxAclConfig="" \
--set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \
--set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
--set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
--set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
deploy/charts/${{ matrix.profile }} \
--debug
- name: run emqx on chart
working-directory: source
if: matrix.discovery == 'dns'
run: |
helm install ${{ matrix.profile }} \
--set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
--set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
--set emqxConfig.EMQX_CLUSTER__DNS__NAME="${{ matrix.profile }}-headless.default.svc.cluster.local" \
--set image.repository=$TARGET \
--set image.pullPolicy=Never \
--set image.tag=$EMQX_TAG \
--set emqxAclConfig="" \
--set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \
--set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
--set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
--set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
deploy/charts/${{ matrix.profile }} \
--debug
- name: waiting emqx started
timeout-minutes: 10
run: |
while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.replicas}')" \
!= "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
echo "==============================";
kubectl get pods;
echo "==============================";
echo "waiting emqx started";
sleep 10;
done
- name: Get Token
timeout-minutes: 1
run: |
kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null &
while
[ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ]
do
echo "waiting emqx"
sleep 1
done
echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV
- name: Check cluster
timeout-minutes: 10
run: |
while
[ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
do
echo "waiting ${{ matrix.profile }} cluster scale"
sleep 1
done
- uses: actions/checkout@v3
with:
repository: emqx/paho.mqtt.testing
ref: develop-5.0
path: paho.mqtt.testing
- name: install pytest
run: |
pip install pytest==7.1.2 pytest-retry
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: run paho test
timeout-minutes: 10
run: |
port_connected () {
local server="$1"
local port="$2"
echo > /dev/tcp/${server}/${port} 2>/dev/null
}
kubectl port-forward service/${{ matrix.profile }} 1883:1883 > /dev/null &
while ! port_connected localhost 1883; do
echo server not listening yet...
sleep 10
done
pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"
- if: failure()
run: kubectl logs -l "app.kubernetes.io/instance=${{ matrix.profile }}" -c emqx --tail=1000

View File

@ -1,16 +0,0 @@
name: Run gitlint
on: [pull_request]
jobs:
run_gitlint:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Run gitlint
shell: bash
run: |
set -ex
docker run --ulimit nofile=1024 -v $(pwd):/repo -w /repo ghcr.io/emqx/gitlint --commits ${{ github.event.pull_request.base.sha }}..$GITHUB_SHA --config .github/workflows/.gitlint

160
.github/workflows/run_helm_tests.yaml vendored Normal file
View File

@ -0,0 +1,160 @@
name: Helm tests
concurrency:
group: helm-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_call:
inputs:
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
# on:
# push:
# branches:
# - master
# - 'ci/**'
# tags:
# - v*
# pull_request:
jobs:
helm_test:
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
env:
EMQX_NAME: ${{ matrix.profile }}
EMQX_TAG: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
REPOSITORY: "emqx/${{ matrix.profile }}"
strategy:
fail-fast: false
matrix:
discovery:
- k8s
- dns
profile:
- emqx
- emqx-enterprise
steps:
- uses: actions/checkout@v3
with:
path: source
- uses: actions/download-artifact@v3
with:
name: "${{ env.EMQX_NAME }}-docker"
path: /tmp
- run: minikube start
- run: |
img="/tmp/${EMQX_NAME}-docker-${EMQX_TAG}.tar.gz"
if stderr=$(minikube image load "${img}" 2>&1 >/dev/null) && test -n "$stderr"; then
echo "${stderr}";
exit 1;
fi
- name: run emqx on chart (k8s)
if: matrix.discovery == 'k8s'
working-directory: source
run: |
helm install ${EMQX_NAME} \
--set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
--set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
--set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${EMQX_NAME}-headless" \
--set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \
--set image.repository=$REPOSITORY \
--set image.pullPolicy=Never \
--set image.tag=$EMQX_TAG \
--set emqxAclConfig="" \
--set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \
--set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
--set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
--set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
deploy/charts/${EMQX_NAME} \
--debug
- name: run emqx on chart (dns)
if: matrix.discovery == 'dns'
working-directory: source
run: |
helm install ${EMQX_NAME} \
--set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
--set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
--set emqxConfig.EMQX_CLUSTER__DNS__NAME="${EMQX_NAME}-headless.default.svc.cluster.local" \
--set image.repository=$REPOSITORY \
--set image.pullPolicy=Never \
--set image.tag=$EMQX_TAG \
--set emqxAclConfig="" \
--set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \
--set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
--set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
--set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
deploy/charts/${EMQX_NAME} \
--debug
- name: waiting emqx started
timeout-minutes: 5
run: |
while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.replicas}')" \
!= "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${EMQX_NAME} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
echo "==============================";
kubectl get pods;
echo "==============================";
echo "waiting emqx started";
sleep 10;
done
- name: Get Token
timeout-minutes: 1
run: |
kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null &
while
[ "$(curl --silent -X 'GET' 'http://127.0.0.1:18083/api/v5/status' | tail -n1)" != "emqx is running" ]
do
echo "waiting emqx"
sleep 1
done
echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV
- name: Check cluster
timeout-minutes: 10
run: |
while
[ "$(curl --silent -H "Authorization: Bearer $TOKEN" -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
do
echo "waiting ${EMQX_NAME} cluster scale"
sleep 1
done
- uses: actions/checkout@v3
with:
repository: emqx/paho.mqtt.testing
ref: develop-5.0
path: paho.mqtt.testing
- name: install pytest
run: |
pip install pytest==7.1.2 pytest-retry
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: run paho test
timeout-minutes: 10
run: |
port_connected () {
local server="$1"
local port="$2"
echo > /dev/tcp/${server}/${port} 2>/dev/null
}
kubectl port-forward service/${EMQX_NAME} 1883:1883 > /dev/null &
while ! port_connected localhost 1883; do
echo server not listening yet...
sleep 10
done
pytest --retries 3 -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"
- if: failure()
run: kubectl logs -l "app.kubernetes.io/instance=${EMQX_NAME}" -c emqx --tail=1000

View File

@ -1,22 +1,23 @@
name: JMeter integration tests name: JMeter integration tests
on: on:
push: workflow_call:
tags: inputs:
- "v5.*" version-emqx:
pull_request: required: true
branches: type: string
- "master" # on:
# push:
# tags:
# - "v5.*"
# pull_request:
# branches:
# - "master"
jobs: jobs:
build_emqx_for_jmeter_tests: jmeter_artifact:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
outputs:
version: ${{ steps.build_docker.outputs.version}}
steps: steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- name: Cache Jmeter - name: Cache Jmeter
id: cache-jmeter id: cache-jmeter
uses: actions/cache@v3 uses: actions/cache@v3
@ -42,21 +43,6 @@ jobs:
with: with:
name: apache-jmeter.tgz name: apache-jmeter.tgz
path: /tmp/apache-jmeter.tgz path: /tmp/apache-jmeter.tgz
- uses: actions/checkout@v3
- name: zip emqx docker image
id: build_docker
if: endsWith(github.repository, 'emqx')
run: |
## TODO: make profile a matrix dimension
PROFILE='emqx'
make "${PROFILE}-docker"
VSN="$(./pkg-vsn.sh $PROFILE)"
echo "version=${VSN}" >> $GITHUB_OUTPUT
docker save -o emqx.tar emqx/emqx:${VSN}
- uses: actions/upload-artifact@v3
with:
name: emqx.tar
path: ./emqx.tar
advanced_feat: advanced_feat:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
@ -70,69 +56,28 @@ jobs:
- mqtt_topic_rewrite - mqtt_topic_rewrite
# - mqtt_retainer # - mqtt_retainer
needs: build_emqx_for_jmeter_tests needs: jmeter_artifact
steps: steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: ./.github/actions/prepare-jmeter
with: with:
name: emqx.tar version-emqx: ${{ inputs.version-emqx }}
path: /tmp
- name: load docker image
run: |
docker load < /tmp/emqx.tar
- name: docker compose up - name: docker compose up
timeout-minutes: 5 timeout-minutes: 5
env:
_EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }}
run: | run: |
docker-compose \ docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
up -d --build up --wait --build
- name: wait docker compose up
timeout-minutes: 5
run: |
while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
sleep 5;
done
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
!= $(docker ps -a --filter name=client | wc -l) ]; do
sleep 1
done
docker ps -a
echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
- uses: actions/checkout@v3 - name: show logs
with: if: failure()
repository: emqx/emqx-fvt
ref: broker-autotest-v5
path: scripts
- uses: actions/setup-java@v3
with:
java-version: '8.0.282' # The JDK version to make available on the path.
java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@v3
with:
name: apache-jmeter.tgz
path: /tmp
- name: install jmeter
timeout-minutes: 10
env:
JMETER_VERSION: 5.4.3
run: | run: |
cd /tmp && tar -xvf apache-jmeter.tgz docker compose \
echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties logs
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
- name: run jmeter - name: run jmeter
run: | run: |
/opt/jmeter/bin/jmeter.sh \ jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \ -Jjmeter.save.saveservice.output_format=xml -n \
-t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \
-Demqx_ip=$HAPROXY_IP \ -Demqx_ip=$HAPROXY_IP \
@ -152,8 +97,6 @@ jobs:
pgsql_authn_authz: pgsql_authn_authz:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env:
_EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }}
strategy: strategy:
fail-fast: false fail-fast: false
@ -168,72 +111,26 @@ jobs:
- pgsql_authn - pgsql_authn
- pgsql_authz - pgsql_authz
needs: build_emqx_for_jmeter_tests needs: jmeter_artifact
steps: steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: ./.github/actions/prepare-jmeter
with: with:
name: emqx.tar version-emqx: ${{ inputs.version-emqx }}
path: /tmp
- name: load docker image
run: |
docker load < /tmp/emqx.tar
- name: docker compose up - name: docker compose up
timeout-minutes: 5 timeout-minutes: 5
env: env:
PGSQL_TAG: ${{ matrix.pgsql_tag }} PGSQL_TAG: ${{ matrix.pgsql_tag }}
run: | run: |
docker-compose \ docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
-f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
up -d --build up --wait --build
- name: wait docker compose up
timeout-minutes: 5
run: |
while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
sleep 5;
done
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
!= $(docker ps -a --filter name=client | wc -l) ]; do
sleep 1
done
docker ps -a
echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql-tls) >> $GITHUB_ENV echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql-tls) >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
ref: broker-autotest-v5
path: scripts
- uses: actions/setup-java@v3
with:
java-version: '8.0.282' # The JDK version to make available on the path.
java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@v3
with:
name: apache-jmeter.tgz
path: /tmp
- name: install jmeter
timeout-minutes: 10
env:
JMETER_VERSION: 5.4.3
run: |
cd /tmp && tar -xvf apache-jmeter.tgz
echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/postgresql-42.2.18.jar https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar
ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
- name: run jmeter - name: run jmeter
run: | run: |
/opt/jmeter/bin/jmeter.sh \ jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \ -Jjmeter.save.saveservice.output_format=xml -n \
-t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \
-Demqx_ip=$HAPROXY_IP \ -Demqx_ip=$HAPROXY_IP \
@ -257,7 +154,7 @@ jobs:
- name: dump docker compose logs - name: dump docker compose logs
if: failure() if: failure()
run: | run: |
docker-compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: always() if: always()
with: with:
@ -277,73 +174,26 @@ jobs:
- mysql_authn - mysql_authn
- mysql_authz - mysql_authz
needs: build_emqx_for_jmeter_tests needs: jmeter_artifact
steps: steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: ./.github/actions/prepare-jmeter
with: with:
name: emqx.tar version-emqx: ${{ inputs.version-emqx }}
path: /tmp
- name: load docker image
run: |
docker load < /tmp/emqx.tar
- name: docker compose up - name: docker compose up
timeout-minutes: 5 timeout-minutes: 5
env: env:
_EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }}
PGSQL_TAG: ${{ matrix.mysql_tag }} PGSQL_TAG: ${{ matrix.mysql_tag }}
run: | run: |
docker-compose \ docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
-f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
up -d --build up --wait --build
- name: wait docker compose up
timeout-minutes: 5
run: |
while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
sleep 5;
done
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
!= $(docker ps -a --filter name=client | wc -l) ]; do
sleep 1
done
docker ps -a
echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql-tls) >> $GITHUB_ENV echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql-tls) >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
ref: broker-autotest-v5
path: scripts
- uses: actions/setup-java@v3
with:
java-version: '8.0.282' # The JDK version to make available on the path.
java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@v3
with:
name: apache-jmeter.tgz
path: /tmp
- name: install jmeter
timeout-minutes: 10
env:
JMETER_VERSION: 5.4.3
run: |
cd /tmp && tar -xvf apache-jmeter.tgz
echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/mysql-connector-java-8.0.16.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
- name: run jmeter - name: run jmeter
run: | run: |
/opt/jmeter/bin/jmeter.sh \ jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \ -Jjmeter.save.saveservice.output_format=xml -n \
-t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \
-Demqx_ip=$HAPROXY_IP \ -Demqx_ip=$HAPROXY_IP \
@ -379,45 +229,19 @@ jobs:
scripts_type: scripts_type:
- jwt_authn - jwt_authn
needs: build_emqx_for_jmeter_tests needs: jmeter_artifact
steps: steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: ./.github/actions/prepare-jmeter
with: with:
name: emqx.tar version-emqx: ${{ inputs.version-emqx }}
path: /tmp
- name: load docker image
run: |
docker load < /tmp/emqx.tar
- name: docker compose up - name: docker compose up
timeout-minutes: 5 timeout-minutes: 5
env:
_EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }}
run: | run: |
docker-compose \ docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
up -d --build up --wait --build
- name: wait docker compose up
timeout-minutes: 5
run: |
while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
sleep 5;
done
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
!= $(docker ps -a --filter name=client | wc -l) ]; do
sleep 1
done
docker ps -a
echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
ref: broker-autotest-v5
path: scripts
- name: run jwks_server - name: run jwks_server
timeout-minutes: 10 timeout-minutes: 10
run: | run: |
@ -426,30 +250,9 @@ jobs:
cd target cd target
docker run --name jwks_server --network emqx_bridge --ip 172.100.239.88 -d -v $(pwd)/jwkserver-0.0.1.jar:/jwks_server/jwkserver-0.0.1.jar --workdir /jwks_server openjdk:8-jdk bash \ docker run --name jwks_server --network emqx_bridge --ip 172.100.239.88 -d -v $(pwd)/jwkserver-0.0.1.jar:/jwks_server/jwkserver-0.0.1.jar --workdir /jwks_server openjdk:8-jdk bash \
-c "java -jar jwkserver-0.0.1.jar" -c "java -jar jwkserver-0.0.1.jar"
- uses: actions/setup-java@v3
with:
java-version: '8.0.282' # The JDK version to make available on the path.
java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@v3
with:
name: apache-jmeter.tgz
path: /tmp
- name: install jmeter
timeout-minutes: 10
env:
JMETER_VERSION: 5.4.3
run: |
cd /tmp && tar -xvf apache-jmeter.tgz
echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
- name: run jmeter - name: run jmeter
run: | run: |
/opt/jmeter/bin/jmeter.sh \ jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \ -Jjmeter.save.saveservice.output_format=xml -n \
-t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \
-Demqx_ip=$HAPROXY_IP \ -Demqx_ip=$HAPROXY_IP \
@ -478,79 +281,30 @@ jobs:
- built_in_database_authn - built_in_database_authn
- built_in_database_authz - built_in_database_authz
needs: build_emqx_for_jmeter_tests needs: jmeter_artifact
steps: steps:
- uses: erlef/setup-beam@v1.15.4
with:
otp-version: 25.3.2
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: actions/download-artifact@v3 - uses: ./.github/actions/prepare-jmeter
with: with:
name: emqx.tar version-emqx: ${{ inputs.version-emqx }}
path: /tmp
- name: load docker image
run: |
docker load < /tmp/emqx.tar
- name: docker compose up - name: docker compose up
timeout-minutes: 5 timeout-minutes: 5
env:
_EMQX_DOCKER_IMAGE_TAG: emqx/emqx:${{ needs.build_emqx_for_jmeter_tests.outputs.version }}
PGSQL_TAG: ${{ matrix.mysql_tag }}
run: | run: |
docker-compose \ docker compose \
-f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
up -d --build up --wait --build
- name: wait docker compose up
timeout-minutes: 5
run: |
while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
sleep 5;
done
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
!= $(docker ps -a --filter name=client | wc -l) ]; do
sleep 1
done
docker ps -a
echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: emqx/emqx-fvt
ref: broker-autotest-v5
path: scripts
- uses: actions/setup-java@v3
with:
java-version: '8.0.282' # The JDK version to make available on the path.
java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@v3
with:
name: apache-jmeter.tgz
path: /tmp
- name: install jmeter
timeout-minutes: 10
env:
JMETER_VERSION: 5.4.3
run: |
cd /tmp && tar -xvf apache-jmeter.tgz
echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/mysql-connector-java-8.0.16.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
- name: run jmeter - name: run jmeter
run: | run: |
/opt/jmeter/bin/jmeter.sh \ jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \ -Jjmeter.save.saveservice.output_format=xml -n \
-t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \ -t scripts/broker-autotest-suite/${{ matrix.scripts_type }}.jmx \
-Demqx_ip=$HAPROXY_IP \ -Demqx_ip=$HAPROXY_IP \
-l jmeter_logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.jtl \ -l jmeter_logs/${{ matrix.scripts_type }}.jtl \
-j jmeter_logs/logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.log -j jmeter_logs/logs/${{ matrix.scripts_type }}.log
- name: check logs - name: check logs
run: | run: |
if cat jmeter_logs/${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then if cat jmeter_logs/${{ matrix.scripts_type }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
echo "check logs failed" echo "check logs failed"
exit 1 exit 1
fi fi
@ -559,11 +313,3 @@ jobs:
with: with:
name: jmeter_logs name: jmeter_logs
path: ./jmeter_logs path: ./jmeter_logs
delete-artifact:
runs-on: ubuntu-22.04
needs: [advanced_feat,pgsql_authn_authz,JWT_authn,mysql_authn_authz,built_in_database_authn_authz]
steps:
- uses: geekyeggo/delete-artifact@v2
with:
name: emqx.tar

View File

@ -4,6 +4,15 @@ concurrency:
group: relup-${{ github.event_name }}-${{ github.ref }} group: relup-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true cancel-in-progress: true
on:
workflow_call:
inputs:
runner:
required: true
type: string
builder:
required: true
type: string
# on: # on:
# push: # push:
# branches: # branches:
@ -14,8 +23,8 @@ concurrency:
jobs: jobs:
relup_test_plan: relup_test_plan:
runs-on: ubuntu-22.04 runs-on: ${{ inputs.runner }}
container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" container: ${{ inputs.builder }}
outputs: outputs:
CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
@ -23,16 +32,18 @@ jobs:
run: run:
shell: bash shell: bash
steps: steps:
- uses: actions/checkout@v3 - uses: AutoModality/action-clean@v1
name: Checkout - uses: actions/download-artifact@v3
with: with:
path: emqx name: emqx-enterprise
fetch-depth: 0 - name: extract artifact
run: |
unzip -o -q emqx-enterprise.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Find versions - name: Find versions
id: find-versions id: find-versions
run: | run: |
set -x set -x
cd emqx
ee_vsn="$(./pkg-vsn.sh enterprise)" ee_vsn="$(./pkg-vsn.sh enterprise)"
old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)"
old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
@ -40,8 +51,6 @@ jobs:
echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT
- name: build emqx - name: build emqx
run: | run: |
set -x
cd emqx
export PROFILE='emqx-enterprise' export PROFILE='emqx-enterprise'
make emqx-enterprise-tgz make emqx-enterprise-tgz
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
@ -49,10 +58,10 @@ jobs:
with: with:
name: emqx_built name: emqx_built
path: | path: |
emqx/_upgrade_base _upgrade_base
emqx/_packages _packages
emqx/scripts scripts
emqx/.ci .ci
relup_test_run: relup_test_run:
needs: needs:
@ -70,8 +79,7 @@ jobs:
run: run:
shell: bash shell: bash
steps: steps:
# setup Erlang to run lux - uses: erlef/setup-beam@v1.16.0
- uses: erlef/setup-beam@v1.15.4
with: with:
otp-version: 25.3.2 otp-version: 25.3.2
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -81,7 +89,7 @@ jobs:
path: lux path: lux
- name: Install lux - name: Install lux
run: | run: |
set -e -u -x set -eu
cd lux cd lux
autoconf autoconf
./configure ./configure
@ -94,10 +102,7 @@ jobs:
path: . path: .
- name: run relup test - name: run relup test
run: | run: |
set -e -x -u set -eux
chmod a+x scripts/**/*.sh
ls -l scripts
ls -l scripts/relup-test
case "$OLD_VSN" in case "$OLD_VSN" in
e*) e*)
export CUR_VSN="$CUR_EE_VSN" export CUR_VSN="$CUR_EE_VSN"

View File

@ -1,156 +1,48 @@
name: Run test case name: Run test case
concurrency: concurrency:
group: test-${{ github.event_name }}-${{ github.ref }} group: test-${{ github.event_name }}-${{ github.sha }}
cancel-in-progress: true cancel-in-progress: true
on: on:
push: workflow_call:
branches: inputs:
- master runner:
- 'ci/**' required: true
tags: type: string
- v* builder:
- e* required: true
pull_request: type: string
ct-matrix:
required: true
type: string
ct-host:
required: true
type: string
ct-docker:
required: true
type: string
# on:
# push:
# branches:
# - master
# - 'ci/**'
# tags:
# - v*
# - e*
# pull_request:
env: env:
IS_CI: "yes" IS_CI: "yes"
jobs: jobs:
build-matrix:
runs-on: ubuntu-22.04
outputs:
prepare: ${{ steps.matrix.outputs.prepare }}
host: ${{ steps.matrix.outputs.host }}
docker: ${{ steps.matrix.outputs.docker }}
runs-on: ${{ steps.runner.outputs.runs-on }}
steps:
- uses: actions/checkout@v3
- name: Build matrix
id: matrix
run: |
APPS="$(./scripts/find-apps.sh --ci)"
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.1-3",
otp: "25.3.2-1",
elixir: "1.14.5"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.1-3",
otp: ["25.3.2-1"][],
elixir: "1.14.5"
})
]
')"
echo "${MATRIX}" | jq
MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT
echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
- name: Choose runner host
id: runner
run: |
RUNS_ON="ubuntu-22.04"
${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64"
echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT
prepare:
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
needs: [build-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/checkout@v3
with:
path: source
- name: get_all_deps
working-directory: source
env:
PROFILE: ${{ matrix.profile }}
run: |
make ensure-rebar3
# fetch all deps and compile
make ${{ matrix.profile }}-compile
make test-compile
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: source.zip
check_examples:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -o -q source.zip
- name: check example config files
env:
PROFILE: ${{ matrix.profile }}
working-directory: source
run: ./scripts/test/check-example-configs.sh
static_checks:
needs:
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -o -q source.zip
- uses: actions/cache@v3
with:
path: "source/emqx_dialyzer_${{ matrix.otp }}_plt"
key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('source/rebar.*', 'source/apps/*/rebar.*', 'source/lib-ee/*/rebar.*') }}
restore-keys: |
rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-
- name: run static checks
env:
PROFILE: ${{ matrix.profile }}
working-directory: source
run: make static_checks
eunit_and_proper: eunit_and_proper:
needs: runs-on: ${{ inputs.runner }}
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} include: ${{ fromJson(inputs.ct-matrix) }}
defaults: defaults:
run: run:
@ -161,16 +53,16 @@ jobs:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-${{ matrix.profile }}-${{ matrix.otp }} name: ${{ matrix.profile }}
path: . - name: extract artifact
- name: unzip source code run: |
run: unzip -o -q source.zip unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# produces eunit.coverdata # produces eunit.coverdata
- name: eunit - name: eunit
env: env:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source
run: make eunit run: make eunit
# produces proper.coverdata # produces proper.coverdata
@ -178,23 +70,19 @@ jobs:
env: env:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
working-directory: source
run: make proper run: make proper
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: coverdata name: coverdata
path: source/_build/test/cover path: _build/test/cover
ct_docker: ct_docker:
needs: runs-on: ${{ inputs.runner }}
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
include: ${{ fromJson(needs.build-matrix.outputs.docker) }} include: ${{ fromJson(inputs.ct-docker) }}
defaults: defaults:
run: run:
@ -204,14 +92,14 @@ jobs:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-${{ matrix.profile }}-${{ matrix.otp }} name: ${{ matrix.profile }}
path: . - name: extract artifact
- name: unzip source code run: |
run: unzip -q source.zip unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
- name: run common tests - name: run common tests
working-directory: source
env: env:
DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
MONGO_TAG: "5" MONGO_TAG: "5"
@ -229,22 +117,19 @@ jobs:
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: coverdata name: coverdata
path: source/_build/test/cover path: _build/test/cover
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: source/_build/test/logs path: _build/test/logs
ct: ct:
needs: runs-on: ${{ inputs.runner }}
- build-matrix
- prepare
runs-on: ${{ needs.build-matrix.outputs.runs-on }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
include: ${{ fromJson(needs.build-matrix.outputs.host) }} include: ${{ fromJson(inputs.ct-host) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
defaults: defaults:
@ -255,14 +140,14 @@ jobs:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-${{ matrix.profile }}-${{ matrix.otp }} name: ${{ matrix.profile }}
path: . - name: extract artifact
- name: unzip source code run: |
run: unzip -q source.zip unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
- name: run common tests - name: run common tests
working-directory: source
env: env:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
SUITEGROUP: ${{ matrix.suitegroup }} SUITEGROUP: ${{ matrix.suitegroup }}
@ -272,58 +157,61 @@ jobs:
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
with: with:
name: coverdata name: coverdata
path: source/_build/test/cover path: _build/test/cover
if-no-files-found: warn # do not fail if no coverdata found if-no-files-found: warn # do not fail if no coverdata found
- uses: actions/upload-artifact@v3 - uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: source/_build/test/logs path: _build/test/logs
make_cover: make_cover:
needs: needs:
- eunit_and_proper - eunit_and_proper
- ct - ct
- ct_docker - ct_docker
runs-on: ubuntu-22.04 runs-on: ${{ inputs.runner }}
container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04" container: ${{ inputs.builder }}
strategy:
fail-fast: false
matrix:
profile:
- emqx-enterprise
steps: steps:
- uses: AutoModality/action-clean@v1 - uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: source-emqx-enterprise-25.3.2-1 name: ${{ matrix.profile }}
path: . - name: extract artifact
- name: unzip source code run: |
run: unzip -q source.zip unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
name: download coverdata name: download coverdata
with: with:
name: coverdata name: coverdata
path: source/_build/test/cover path: _build/test/cover
- name: make cover - name: make cover
working-directory: source
env: env:
PROFILE: emqx-enterprise PROFILE: emqx-enterprise
run: make cover run: make cover
- name: send to coveralls - name: send to coveralls
working-directory: source
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PROFILE: emqx-enterprise PROFILE: emqx-enterprise
run: make coveralls run: make coveralls
- name: get coveralls logs - name: get coveralls logs
working-directory: source
if: failure() if: failure()
run: cat rebar3.crashdump run: cat rebar3.crashdump
# do this in a separate job # do this in a separate job
upload_coverdata: upload_coverdata:
needs: make_cover needs: make_cover
runs-on: ubuntu-22.04 runs-on: ${{ inputs.runner }}
steps: steps:
- name: Coveralls Finished - name: Coveralls Finished
env: env:

View File

@ -1,19 +0,0 @@
name: Shellcheck
on:
pull_request:
jobs:
shellcheck:
runs-on: ubuntu-22.04
steps:
- name: Checkout source code
uses: actions/checkout@v3
- name: Install shellcheck
run: |
sudo apt-get update
sudo apt install shellcheck
- name: Run shellcheck
run: |
./scripts/shellcheck.sh
echo "success"

46
.github/workflows/spellcheck.yaml vendored Normal file
View File

@ -0,0 +1,46 @@
name: Build slim packages
concurrency:
group: spellcheck-${{ github.event_name }}-${{ github.sha }}
cancel-in-progress: true
on:
workflow_call:
inputs:
runner:
required: true
type: string
# on:
# push:
# branches:
# - master
# - release-51
# pull_request:
# # GitHub pull_request action is by default triggered when
# # opened reopened or synchronize,
# # we add labeled and unlabeled to the list because
# # the mac job dpends on the PR having a 'Mac' label
# types:
# - labeled
# - unlabeled
# - opened
# - reopened
# - synchronize
# workflow_dispatch:
jobs:
spellcheck:
strategy:
matrix:
profile:
- emqx
- emqx-enterprise
runs-on: ${{ inputs.runner }}
steps:
- uses: actions/download-artifact@v3
with:
name: "${{ matrix.profile }}_schema_dump"
path: /tmp/
- name: Run spellcheck
run: |
bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json

59
.github/workflows/static_checks.yaml vendored Normal file
View File

@ -0,0 +1,59 @@
name: Static checks
concurrency:
group: static-checks-${{ github.event_name }}-${{ github.sha }}
cancel-in-progress: true
on:
workflow_call:
inputs:
runner:
required: true
type: string
builder:
required: true
type: string
ct-matrix:
required: true
type: string
# on:
# push:
# branches:
# - master
# - 'ci/**'
# tags:
# - v*
# - e*
# pull_request:
env:
IS_CI: "yes"
jobs:
static_checks:
runs-on: ${{ inputs.runner }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(inputs.ct-matrix) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/cache@v3
with:
path: "emqx_dialyzer_${{ matrix.otp }}_plt"
key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*', 'lib-ee/*/rebar.*') }}
restore-keys: |
rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-
- name: run static checks
env:
PROFILE: ${{ matrix.profile }}
run: make static_checks

14
build
View File

@ -361,10 +361,10 @@ make_tgz() {
log "Archive sha256sum: $(cat "${target}.sha256")" log "Archive sha256sum: $(cat "${target}.sha256")"
} }
trap docker_cleanup EXIT
docker_cleanup() { docker_cleanup() {
rm -f ./.dockerignore >/dev/null rm -f ./.dockerignore >/dev/null
# shellcheck disable=SC2015
[ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true
} }
## Build the default docker image based on debian 11. ## Build the default docker image based on debian 11.
@ -384,7 +384,14 @@ make_docker() {
if [[ "$PROFILE" = *enterprise* ]]; then if [[ "$PROFILE" = *enterprise* ]]; then
extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit' extra_deps='libsasl2-2,libsasl2-modules-gssapi-mit'
fi fi
echo '_build' >> ./.dockerignore # shellcheck disable=SC2015
[ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true
trap docker_cleanup EXIT
{
echo '/_build'
echo '/deps'
echo '/*.lock'
} >> ./.dockerignore
set -x set -x
docker build --no-cache --pull \ docker build --no-cache --pull \
--build-arg BUILD_FROM="${EMQX_BUILDER}" \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \
@ -394,6 +401,7 @@ make_docker() {
--tag "${EMQX_IMAGE_TAG}" \ --tag "${EMQX_IMAGE_TAG}" \
-f "${EMQX_DOCKERFILE}" . -f "${EMQX_DOCKERFILE}" .
[[ "${DEBUG:-}" -eq 1 ]] || set +x [[ "${DEBUG:-}" -eq 1 ]] || set +x
echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag
} }
function join { function join {