yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci: call more workflows

This commit is contained in:
Ivan Dyachkov 2023-07-20 17:04:21 +02:00
parent eaca8e2d9a
commit ab0aedd501
6 changed files with 212 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
.github/workflows/_pr_entrypoint.yaml vendored
View File

@ -11,9 +11,11 @@ jobs:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
host: ${{ steps.matrix.outputs.host }}
docker: ${{ steps.matrix.outputs.docker }}
ct-matrix: ${{ steps.matrix.outputs.matrix }}
ct-host: ${{ steps.matrix.outputs.host }}
ct-docker: ${{ steps.matrix.outputs.docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
steps:
- uses: actions/checkout@v3
@ -39,12 +41,14 @@ jobs:
]
')"
echo "${MATRIX}" | jq
MATRIX_PREPARED="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "matrix=${MATRIX_PREPARED}" | tee -a $GITHUB_OUTPUT
echo "ct-host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT
echo "ct-docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT
CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
CT_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')"
CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
echo "ct-host=${CT_HOST}" | tee -a $GITHUB_OUTPUT
echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT
echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT
echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
compile:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
@ -55,9 +59,6 @@ jobs:
profile:
- emqx
- emqx-enterprise
outputs:
version-emqx: ${{ steps.compile.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.compile.outputs.version-emqx-enterprise }}
steps:
- uses: actions/checkout@v3
@ -71,8 +72,6 @@ jobs:
PROFILE: ${{ matrix.profile }}
run: |
make $PROFILE
./pkg-vsn.sh $PROFILE > VSN
echo "version-$PROFILE=$(cat VSN)" | tee -a $GITHUB_OUTPUT
zip -ryq $PROFILE.zip .
- uses: actions/upload-artifact@v3
with:
@ -80,27 +79,45 @@ jobs:
path: ${{ matrix.profile }}.zip
retention-days: 1
# run_test_cases:
# needs: [sanity-checks, compile]
# uses: ./.github/workflows/run_test_cases.yaml
# with:
# matrix: ${{ needs.sanity-checks.outputs.matrix }}
# ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
# ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
run_test_cases:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/run_test_cases.yaml
with:
ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
# build_slim_packages:
# needs: [sanity-checks]
# uses: ./.github/workflows/build_slim_packages.yaml
build_slim_packages:
needs:
- sanity-checks
uses: ./.github/workflows/build_slim_packages.yaml
call_conf_tests:
build_docker_for_test:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/build_docker_for_test.yaml
with:
version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
spellcheck:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/spellcheck.yaml
run_conf_tests:
needs:
- sanity-checks
- compile
uses: ./.github/workflows/run_conf_tests.yaml
# check_deps_integrity:
# needs: [sanity-checks]
# uses: ./.github/workflows/check_deps_integrity.yaml
check_deps_integrity:
needs: [sanity-checks]
uses: ./.github/workflows/check_deps_integrity.yaml
# elixir_release:
# needs: [sanity-checks]

90
.github/workflows/build_docker_for_test.yaml vendored Normal file
View File

@ -0,0 +1,90 @@
name: Build docker image for test
concurrency:
group: docker-test-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_call:
inputs:
version-emqx:
required: true
type: string
version-emqx-enterprise:
required: true
type: string
# on:
# push:
# branches:
# - master
# - release-51
# pull_request:
# # GitHub pull_request action is by default triggered when
# # opened reopened or synchronize,
# # we add labeled and unlabeled to the list because
# # the mac job dpends on the PR having a 'Mac' label
# types:
# - labeled
# - unlabeled
# - opened
# - reopened
# - synchronize
# workflow_dispatch:
jobs:
docker:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
env:
EMQX_NAME: ${{ matrix.profile[0] }}
EMQX_IMAGE_TAG: emqx/${{ matrix.profile[0] }}:test
EMQX_IMAGE_OLD_VERSION_TAG: emqx/${{ matrix.profile[0] }}:${{ matrix.profile[1] }}
PKG_VSN: ${{ matrix.profile[0] == 'emqx' && inputs.version-emqx || inputs.version-emqx-enterprise }}
strategy:
fail-fast: false
matrix:
profile:
- ["emqx", "5.0.16"]
- ["emqx-enterprise", "5.0.1"]
steps:
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile[0] }}
- name: extract artifact
run: |
unzip -o -q $EMQX_NAME.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: build and export to Docker
run: |
docker build -t emqx/$EMQX_NAME:test --build-arg EMQX_NAME=$EMQX_NAME -f ./deploy/docker/Dockerfile .
- name: smoke test
run: |
CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
docker stop $CID
- name: dashboard tests
working-directory: ./scripts/ui-tests
run: |
set -eu
docker compose up --abort-on-container-exit --exit-code-from selenium
- name: test two nodes cluster with proto_dist=inet_tls in docker
run: |
./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
# cleanup
./scripts/test/start-two-nodes-in-docker.sh -c
- name: export docker image
run: |
docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
- uses: actions/upload-artifact@v3
with:
name: "${{ matrix.profile[0] }}-docker"
path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"
- name: cleanup
if: always()
working-directory: ./scripts/ui-tests
run: |
docker compose rm -fs

122
.github/workflows/build_slim_packages.yaml vendored
View File

@ -27,6 +27,8 @@ on:
jobs:
linux:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
env:
EMQX_NAME: ${{ matrix.profile[0] }}
strategy:
fail-fast: false
@ -48,13 +50,10 @@ jobs:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: prepare
run: |
echo "EMQX_NAME=${{ matrix.profile[0] }}" >> $GITHUB_ENV
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
- name: build and test tgz package
if: matrix.profile[3] == 'erlang'
run: |
@ -131,118 +130,3 @@ jobs:
with:
name: windows
path: _packages/${{ matrix.profile }}/*
mac:
strategy:
fail-fast: false
matrix:
profile:
- emqx
- emqx-enterprise
otp:
- 25.3.2-1
os:
- macos-11
- macos-12-arm64
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: prepare
run: |
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
- uses: ./.github/actions/package-macos
with:
profile: ${{ matrix.profile }}
otp: ${{ matrix.otp }}
os: ${{ matrix.os }}
apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.os }}
path: _packages/**/*
docker:
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
strategy:
fail-fast: false
matrix:
profile:
- ["emqx", "5.0.16"]
- ["emqx-enterprise", "5.0.1"]
steps:
- uses: actions/checkout@v3
- name: prepare
run: |
EMQX_NAME=${{ matrix.profile[0] }}
PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}
EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test
EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }}
echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV
echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV
echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV
- uses: docker/setup-buildx-action@v2
- name: build and export to Docker
uses: docker/build-push-action@v4
with:
context: .
file: ./deploy/docker/Dockerfile
load: true
tags: ${{ env.EMQX_IMAGE_TAG }}
build-args: |
EMQX_NAME=${{ env.EMQX_NAME }}
- name: smoke test
run: |
CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
docker stop $CID
- name: dashboard tests
working-directory: ./scripts/ui-tests
run: |
set -eu
docker compose up --abort-on-container-exit --exit-code-from selenium
- name: test two nodes cluster with proto_dist=inet_tls in docker
run: |
./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
# cleanup
./scripts/test/start-two-nodes-in-docker.sh -c
- name: export docker image
run: |
docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz
- uses: actions/upload-artifact@v3
with:
name: "${{ matrix.profile[0] }}-docker"
path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz"
- name: cleanup
if: always()
working-directory: ./scripts/ui-tests
run: |
docker compose rm -fs
spellcheck:
needs: linux
strategy:
matrix:
profile:
- emqx
- emqx-enterprise
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
steps:
- uses: actions/download-artifact@v3
name: Download schema dump
with:
name: "${{ matrix.profile }}_schema_dump"
path: /tmp/
- name: Run spellcheck
run: |
bash /tmp/scripts/spellcheck/spellcheck.sh /tmp/_build/docgen/${{ matrix.profile }}/schema-en.json

2
.github/workflows/run_conf_tests.yaml vendored
View File

@ -47,7 +47,7 @@ jobs:
env:
PROFILE: ${{ matrix.profile }}
run: ./scripts/conf-test/run.sh
- name: print_erlang_log
- name: print erlang log
if: failure()
run: |
cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*

51
.github/workflows/run_test_cases.yaml vendored
View File

@ -7,7 +7,7 @@ concurrency:
on:
workflow_call:
inputs:
matrix:
ct-matrix:
required: true
type: string
ct-host:
@ -36,16 +36,17 @@ jobs:
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(inputs.matrix) }}
include: ${{ fromJson(inputs.ct-matrix) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}-${{ github.sha }}
path: ${{ matrix.profile }}.zip
- name: unzip source code
run: unzip -o -q ${{ matrix.profile }}.zip
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/cache@v3
with:
path: "emqx_dialyzer_${{ matrix.otp }}_plt"
@ -62,7 +63,7 @@ jobs:
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(inputs.matrix) }}
include: ${{ fromJson(inputs.ct-matrix) }}
defaults:
run:
@ -73,10 +74,11 @@ jobs:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}-${{ github.sha }}
path: ${{ matrix.profile }}.zip
- name: unzip source code
run: unzip -o -q ${{ matrix.profile }}.zip
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# produces eunit.coverdata
- name: eunit
env:
@ -111,10 +113,11 @@ jobs:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}-${{ github.sha }}
path: ${{ matrix.profile }}.zip
- name: unzip source code
run: unzip -o -q ${{ matrix.profile }}.zip
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
- name: run common tests
@ -158,10 +161,11 @@ jobs:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}-${{ github.sha }}
path: ${{ matrix.profile }}.zip
- name: unzip source code
run: unzip -o -q ${{ matrix.profile }}.zip
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
- name: run common tests
@ -198,10 +202,11 @@ jobs:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}-${{ github.sha }}
path: ${{ matrix.profile }}.zip
- name: unzip source code
run: unzip -o -q ${{ matrix.profile }}.zip
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/download-artifact@v3
name: download coverdata

45
.github/workflows/spellcheck.yaml vendored Normal file
View File

@ -0,0 +1,45 @@
name: Build slim packages
concurrency:
group: spellcheck-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_call:
# on:
# push:
# branches:
# - master
# - release-51
# pull_request:
# # GitHub pull_request action is by default triggered when
# # opened reopened or synchronize,
# # we add labeled and unlabeled to the list because
# # the mac job dpends on the PR having a 'Mac' label
# types:
# - labeled
# - unlabeled
# - opened
# - reopened
# - synchronize
# workflow_dispatch:
jobs:
spellcheck:
strategy:
matrix:
profile:
- emqx
- emqx-enterprise
runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
steps:
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Run spellcheck
run: |
./scripts/spellcheck/spellcheck.sh _build/docgen/${{ matrix.profile }}/schema-en.json